<commit_before>import hypothesis
import hypothesis.strategies

import numpy
import scipy.sparse

from wide_product import wide_product

FINITE_FLOAT = hypothesis.strategies.floats(
    allow_nan=False,
    allow_infinity=False,
)

DIMENSION = hypothesis.strategies.integers(
    min_value=1,
    max_value=2**30,
)

SENSIBLE_DIMENSION = hypothesis.strategies.integers(
    min_value=1,
    max_value=1000,
)

def get_sparse_matrix(shape, data):
    rows, cols = shape

    element_data = numpy.array(data.draw(hypothesis.strategies.lists(
        FINITE_FLOAT,
        min_size=0,
        max_size=rows * cols,
    )))

    if len(element_data) == 0:
        return scipy.sparse.csr_matrix(shape)

    locations = numpy.array(data.draw(hypothesis.strategies.lists(
        hypothesis.strategies.tuples(
            hypothesis.strategies.integers(0, rows - 1),
            hypothesis.strategies.integers(0, cols - 1),
        ),
        min_size=len(element_data),
        max_size=len(element_data),
        unique=True,
    )), dtype='int32')

    return scipy.sparse.coo_matrix(
        (element_data, (locations[:, 0], locations[:, 1])),
        shape=shape,
    ).tocsr()

@hypothesis.given(
    x=FINITE_FLOAT,
    y=FINITE_FLOAT,
)
def test_scalar_wide_product_equivalent_to_multiplication(x, y):
    product = wide_product([[x]], [[y]])
    assert product.shape == (1, 1)
    assert product[0, 0] == x * y

@hypothesis.given(
    a_width=DIMENSION,
    b_width=DIMENSION,
    height=SENSIBLE_DIMENSION,
)
def test_shapes_of_empty_sparse_matrices_are_correct(a_width, b_width, height):
    a = scipy.sparse.csr_matrix((height, a_width))
    b = scipy.sparse.csr_matrix((height, b_width))

    product = wide_product(a, b).tocsr()

    assert product.shape == (height, a_width * b_width)

@hypothesis.given(
    a_width=SENSIBLE_DIMENSION,
    b_width=SENSIBLE_DIMENSION,
    height_top=SENSIBLE_DIMENSION,
    height_bottom=SENSIBLE_DIMENSION,
    data=hypothesis.strategies.data(),
)
def test_horizontal_stacking(a_width, b_width, height_top, height_bottom, data):
    a_top = get_sparse_matrix((height_top, a_width), data)
    b_top = get_sparse_matrix((height_top, b_width), data)
    a_bottom = get_sparse_matrix((height_bottom, a_width), data)
    b_bottom = get_sparse_matrix((height_bottom, b_width), data)

    top_product = wide_product(a_top, b_top)
    bottom_product = wide_product(a_bottom, b_bottom)
    stacked_product = scipy.sparse.vstack((top_product, bottom_product))

    a = scipy.sparse.vstack((a_top, a_bottom))
    b = scipy.sparse.vstack((b_top, b_bottom))

    full_product = wide_product(a, b)

    assert full_product == stacked_product
<commit_msg>test array equality in backwards-compatible way<commit_after>import hypothesis
import hypothesis.strategies

import numpy
import scipy.sparse

from wide_product import wide_product

FINITE_FLOAT = hypothesis.strategies.floats(
    allow_nan=False,
    allow_infinity=False,
)

DIMENSION = hypothesis.strategies.integers(
    min_value=1,
    max_value=2**30,
)

SENSIBLE_DIMENSION = hypothesis.strategies.integers(
    min_value=1,
    max_value=1000,
)

def get_sparse_matrix(shape, data):
    rows, cols = shape

    element_data = numpy.array(data.draw(hypothesis.strategies.lists(
        FINITE_FLOAT,
        min_size=0,
        max_size=rows * cols,
    )))

    if len(element_data) == 0:
        return scipy.sparse.csr_matrix(shape)

    locations = numpy.array(data.draw(hypothesis.strategies.lists(
        hypothesis.strategies.tuples(
            hypothesis.strategies.integers(0, rows - 1),
            hypothesis.strategies.integers(0, cols - 1),
        ),
        min_size=len(element_data),
        max_size=len(element_data),
        unique=True,
    )), dtype='int32')

    return scipy.sparse.coo_matrix(
        (element_data, (locations[:, 0], locations[:, 1])),
        shape=shape,
    ).tocsr()

@hypothesis.given(
    x=FINITE_FLOAT,
    y=FINITE_FLOAT,
)
def test_scalar_wide_product_equivalent_to_multiplication(x, y):
    product = wide_product([[x]], [[y]])
    assert product.shape == (1, 1)
    assert product[0, 0] == x * y

@hypothesis.given(
    a_width=DIMENSION,
    b_width=DIMENSION,
    height=SENSIBLE_DIMENSION,
)
def test_shapes_of_empty_sparse_matrices_are_correct(a_width, b_width, height):
    a = scipy.sparse.csr_matrix((height, a_width))
    b = scipy.sparse.csr_matrix((height, b_width))

    product = wide_product(a, b).tocsr()

    assert product.shape == (height, a_width * b_width)

@hypothesis.given(
    a_width=SENSIBLE_DIMENSION,
    b_width=SENSIBLE_DIMENSION,
    height_top=SENSIBLE_DIMENSION,
    height_bottom=SENSIBLE_DIMENSION,
    data=hypothesis.strategies.data(),
)
def test_horizontal_stacking(a_width, b_width, height_top, height_bottom, data):
    a_top = get_sparse_matrix((height_top, a_width), data)
    b_top = get_sparse_matrix((height_top, b_width), data)
    a_bottom = get_sparse_matrix((height_bottom, a_width), data)
    b_bottom = get_sparse_matrix((height_bottom, b_width), data)

    top_product = wide_product(a_top, b_top)
    bottom_product = wide_product(a_bottom, b_bottom)
    stacked_product = scipy.sparse.vstack((top_product, bottom_product))

    a = scipy.sparse.vstack((a_top, a_bottom))
    b = scipy.sparse.vstack((b_top, b_bottom))

    full_product = wide_product(a, b)

    x = (full_product != stacked_product).todense().ravel()

    assert not numpy.any(x)
<|endoftext|>


<commit_before>from __future__ import division

import sys
import logging
from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter
from itertools import groupby

import numpy as NP
import pandas as PD
import scipy.signal

from iaga2hdf import read_hdf, write_hdf
from ..util.nan import nan_interp
from ..util.signal import lp_fir_filter

logger = logging.getLogger('pyrsss.mag.process_hdf')

def minute_interval_filter_firwin():
    """
    Synthesize and return impulse response of minute interval filter
    for analysis of magnetometer data. Design a filter using the
    window method. The filter is designed to stop from 0 to 0.75 mHz
    and pass from 1 mHz to Nyquist.

    For reference: this is the filter used to process the data for the
    Space Weather 2017 paper.

    fs_1m = 1./60
    bands_1m = [0, 1e-3 - 0.25e-3, 1e-3, fs_1m/2]
    desired_1m = [0, 1]

    _ = fir_response(h_1m, bands_1m, desired_1m, Hz=fs_1m)

    FIR filter is linear phase type 1

    band 0 abs deviations from 0:
    min = 4.466e-05  (db=-87.000846)
    med = -4.218e-03
    max = 3.027e-02 (db=-30.379491

    band 1 abs deviations from 1:
    min = 4.412e-06  (db=-107.106525)
    med = 2.398e-03
    std = 4.059e-03
    max = 2.982e-02 (db=-30.509428
    """
    Ts = 60
    fs = 1. / Ts
    fn = fs / 2
    width = 0.25e-3
    cutoff = 1e-3
    ripple = 30
    numtaps, beta = scipy.signal.kaiserord(ripple, width / fn)
    if numtaps % 2 == 0:
        numtaps += 1
    return scipy.signal.firwin(numtaps,
                               cutoff - width/2,
                               width=width,
                               pass_zero=False,
                               nyq=fn)

def minute_interval_filter(N_remez=201):
    """
    Synthesize and return impulse response of minute interval filter
    for analysis of magnetometer data. Design a length *N_remez*
    min-max optimal filter. The filter is designed to stop from 0 to
    0.75 mHz and pass from 1 mHz to Nyquist.

    fs_1m = 1./60
    bands_1m = [0, 1e-3 - 0.25e-3, 1e-3, fs_1m/2]
    desired_1m = [0, 1]

    _ = fir_response(h_1m, bands_1m, desired_1m, Hz=fs_1m)

    FIR filter is linear phase type 1

    band 0 abs deviations from 0:
    min = 1.487e-05  (db=-96.553225)
    med = -1.102e-03
    max = 1.569e-03 (db=-56.089238

    band 1 abs deviations from 1:
    min = 1.018e-06  (db=-119.848775)
    med = -1.066e-05
    std = 4.820e-04
    max = 1.571e-03 (db=-56.074809
    """
    return scipy.signal.remez(N_remez,
                              [0, 0.75e-3, 1e-3, 1./60/2],
                              [0, 1],
                              [1, 1],
                              Hz=1./60)

def second_interval_filter(N_remez=4001):
    """
    Synthesize and return impulse response of second interval filter
    for analysis of magnetometer data. Design a length *N_remez*
    min-max optimal filter. The filter is designed to stop from 0 to
    0.25 mHz, pass from 1 to 100 mHz, and stop from 101 mHz to
    Nyquist.

    fs_1m = 1./60
    bands_1m = [0, 1e-3 - 0.25e-3, 1e-3, fs_1m/2]
    desired_1m = [0, 1]

    _ = fir_response(h_1m, bands_1m, desired_1m, Hz=fs_1m)

    FIR filter is linear phase type 1

    band 0 abs deviations from 0:
    min = 6.872e-05  (db=-83.257739)
    med = -2.271e-03
    max = 4.902e-03 (db=-46.192275

    band 1 abs deviations from 1:
    min = 2.323e-07  (db=-132.678419)
    med = -6.093e-07
    std = 5.871e-04
    max = 2.349e-03 (db=-52.584106

    band 2 abs deviations from 0:
    min = 1.430e-07  (db=-136.893586)
    med = -1.353e-03
    max = 2.853e-03 (db=-50.895031
    """
    return scipy.signal.remez(N_remez,
                              [0, 0.25e-3, 1e-3, 100e-3, 101e-3, 0.5],
                              [0, 1, 0],
                              [1, 1, 1],
                              Hz=1)

def consecutive_nans(x, y):
    """
    Return the maximum number of nans encountered in the logical or of
    the *x* and *y* time series.
    """
    nan = NP.isnan(x) | NP.isnan(y)
    lengths = []
    for key, group in groupby(nan):
        if key == True:
            lengths.append(len(list(group)))
    if lengths:
        return max(lengths)
    return 0

def process_timeseries(dt,
                       Bx,
                       By,
                       c1='Bx',
                       c2='By',
                       remove_mean=True):
    """
    Process surface magnetic field measurement time series with
    indices *dt* and components *Bx* and *By*. Output a
    :class:`DataFrame` with columns *c1* and *c2* associated with the
    processed output time series. If *remove_mean*, remove the mean
    from the output time series.
    """
    n = consecutive_nans(Bx, By)
    interval = (dt[1] - dt[0]).total_seconds()
    if n > 0:
        logger.warning('longest contiguous gap = {:.2f} minutes'.format(n * interval / 60))
    # fill data gaps via linear interpolation
    Bx = nan_interp(Bx)
    By = nan_interp(By)
    # apply 1 - 100 mHz bandpass filter
    if interval == 1.0:
        h = second_interval_filter()
    elif interval == 60.0:
        h = minute_interval_filter()
    else:
        raise ValueError('1 to 100 mHz filter not yet synthesized for {} s interval data'.format(interval))
    Bx_filtered, dt_filtered = lp_fir_filter(h, Bx, mode='valid', index=dt)
    By_filtered = lp_fir_filter(h, By, mode='valid')
    # remove mean
    if remove_mean:
        Bx_filtered -= NP.mean(Bx_filtered)
        By_filtered -= NP.mean(By_filtered)
    # build DataFrame and store to disk
    return PD.DataFrame(index=dt_filtered,
                        data={c1: Bx_filtered,
                              c2: By_filtered})

def process(hdf_fname,
            source_key='B_raw',
            key='B',
            he=False,
            remove_mean=True):
    """
    Process the magnetic field columns of *hdf_fname*, applying
    pre-processing (nan interpolation) and a band-pass filter. Look
    for input at *source_key* and store output under identifier
    *key*. If *he*, process the H and E magnetic field components. If
    *remove_mean*, remove the mean from each column.
    """
    logger.info('processing {}'.format(hdf_fname))
    df_raw, header = read_hdf(hdf_fname, source_key)
    dt = [PD.to_datetime(x).to_pydatetime() for x in df_raw.index.values]
    Bx_raw = df_raw['Bx'].values * 1e-9
    By_raw = df_raw['By'].values * 1e-9
    df_filtered = process_timeseries(dt,
                                     Bx_raw,
                                     By_raw,
                                     remove_mean=remove_mean)
    if he:
        Bh_raw = df_raw['Bh'].values * 1e-9
        Be_raw = df_raw['Be'].values * 1e-9
        df_he_filtered = process_timeseries(dt,
                                            Bh_raw,
                                            Be_raw,
                                            c1='Bh',
                                            c2='Be',
                                            remove_mean=remove_mean)
        df_filtered = df_filtered.join(df_he_filtered)
    write_hdf(hdf_fname, df_filtered, key, header)
    return hdf_fname

def main(argv=None):
    if argv is None:
        argv = sys.argv

    parser = ArgumentParser('',
                            formatter_class=ArgumentDefaultsHelpFormatter)
    parser.add_argument('hdf_fnames',
                        type=str,
                        nargs='*',
                        metavar='hdf_fname',
                        help='HDF file record to process')
    parser.add_argument('--source-key',
                        '-s',
                        type=str,
                        default='B_raw',
                        help='')
    parser.add_argument('--key',
                        '-k',
                        type=str,
                        default='B',
                        help='key to associate with the processed records')
    parser.add_argument('--he',
                        action='store_true',
                        help='include results in HE coordinate')
    args = parser.parse_args(argv[1:])

    for hdf_fname in args.hdf_fnames:
        process(hdf_fname,
                source_key=args.source_key,
                key=args.key,
                he=args.he)

if __name__ == '__main__':
    logging.basicConfig(level=logging.INFO)
    sys.exit(main())
<commit_msg>Added usage message.<commit_after>from __future__ import division

import sys
import logging
from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter
from itertools import groupby

import numpy as NP
import pandas as PD
import scipy.signal

from iaga2hdf import read_hdf, write_hdf
from ..util.nan import nan_interp
from ..util.signal import lp_fir_filter

logger = logging.getLogger('pyrsss.mag.process_hdf')

def minute_interval_filter_firwin():
    """
    Synthesize and return impulse response of minute interval filter
    for analysis of magnetometer data. Design a filter using the
    window method. The filter is designed to stop from 0 to 0.75 mHz
    and pass from 1 mHz to Nyquist.

    For reference: this is the filter used to process the data for the
    Space Weather 2017 paper.

    fs_1m = 1./60
    bands_1m = [0, 1e-3 - 0.25e-3, 1e-3, fs_1m/2]
    desired_1m = [0, 1]

    _ = fir_response(h_1m, bands_1m, desired_1m, Hz=fs_1m)

    FIR filter is linear phase type 1

    band 0 abs deviations from 0:
    min = 4.466e-05  (db=-87.000846)
    med = -4.218e-03
    max = 3.027e-02 (db=-30.379491

    band 1 abs deviations from 1:
    min = 4.412e-06  (db=-107.106525)
    med = 2.398e-03
    std = 4.059e-03
    max = 2.982e-02 (db=-30.509428
    """
    Ts = 60
    fs = 1. / Ts
    fn = fs / 2
    width = 0.25e-3
    cutoff = 1e-3
    ripple = 30
    numtaps, beta = scipy.signal.kaiserord(ripple, width / fn)
    if numtaps % 2 == 0:
        numtaps += 1
    return scipy.signal.firwin(numtaps,
                               cutoff - width/2,
                               width=width,
                               pass_zero=False,
                               nyq=fn)

def minute_interval_filter(N_remez=201):
    """
    Synthesize and return impulse response of minute interval filter
    for analysis of magnetometer data. Design a length *N_remez*
    min-max optimal filter. The filter is designed to stop from 0 to
    0.75 mHz and pass from 1 mHz to Nyquist.

    fs_1m = 1./60
    bands_1m = [0, 1e-3 - 0.25e-3, 1e-3, fs_1m/2]
    desired_1m = [0, 1]

    _ = fir_response(h_1m, bands_1m, desired_1m, Hz=fs_1m)

    FIR filter is linear phase type 1

    band 0 abs deviations from 0:
    min = 1.487e-05  (db=-96.553225)
    med = -1.102e-03
    max = 1.569e-03 (db=-56.089238

    band 1 abs deviations from 1:
    min = 1.018e-06  (db=-119.848775)
    med = -1.066e-05
    std = 4.820e-04
    max = 1.571e-03 (db=-56.074809
    """
    return scipy.signal.remez(N_remez,
                              [0, 0.75e-3, 1e-3, 1./60/2],
                              [0, 1],
                              [1, 1],
                              Hz=1./60)

def second_interval_filter(N_remez=4001):
    """
    Synthesize and return impulse response of second interval filter
    for analysis of magnetometer data. Design a length *N_remez*
    min-max optimal filter. The filter is designed to stop from 0 to
    0.25 mHz, pass from 1 to 100 mHz, and stop from 101 mHz to
    Nyquist.

    fs_1m = 1./60
    bands_1m = [0, 1e-3 - 0.25e-3, 1e-3, fs_1m/2]
    desired_1m = [0, 1]

    _ = fir_response(h_1m, bands_1m, desired_1m, Hz=fs_1m)

    FIR filter is linear phase type 1

    band 0 abs deviations from 0:
    min = 6.872e-05  (db=-83.257739)
    med = -2.271e-03
    max = 4.902e-03 (db=-46.192275

    band 1 abs deviations from 1:
    min = 2.323e-07  (db=-132.678419)
    med = -6.093e-07
    std = 5.871e-04
    max = 2.349e-03 (db=-52.584106

    band 2 abs deviations from 0:
    min = 1.430e-07  (db=-136.893586)
    med = -1.353e-03
    max = 2.853e-03 (db=-50.895031
    """
    return scipy.signal.remez(N_remez,
                              [0, 0.25e-3, 1e-3, 100e-3, 101e-3, 0.5],
                              [0, 1, 0],
                              [1, 1, 1],
                              Hz=1)

def consecutive_nans(x, y):
    """
    Return the maximum number of nans encountered in the logical or of
    the *x* and *y* time series.
    """
    nan = NP.isnan(x) | NP.isnan(y)
    lengths = []
    for key, group in groupby(nan):
        if key == True:
            lengths.append(len(list(group)))
    if lengths:
        return max(lengths)
    return 0

def process_timeseries(dt,
                       Bx,
                       By,
                       c1='Bx',
                       c2='By',
                       remove_mean=True):
    """
    Process surface magnetic field measurement time series with
    indices *dt* and components *Bx* and *By*. Output a
    :class:`DataFrame` with columns *c1* and *c2* associated with the
    processed output time series. If *remove_mean*, remove the mean
    from the output time series.
    """
    n = consecutive_nans(Bx, By)
    interval = (dt[1] - dt[0]).total_seconds()
    if n > 0:
        logger.warning('longest contiguous gap = {:.2f} minutes'.format(n * interval / 60))
    # fill data gaps via linear interpolation
    Bx = nan_interp(Bx)
    By = nan_interp(By)
    # apply 1 - 100 mHz bandpass filter
    if interval == 1.0:
        h = second_interval_filter()
    elif interval == 60.0:
        h = minute_interval_filter()
    else:
        raise ValueError('1 to 100 mHz filter not yet synthesized for {} s interval data'.format(interval))
    Bx_filtered, dt_filtered = lp_fir_filter(h, Bx, mode='valid', index=dt)
    By_filtered = lp_fir_filter(h, By, mode='valid')
    # remove mean
    if remove_mean:
        Bx_filtered -= NP.mean(Bx_filtered)
        By_filtered -= NP.mean(By_filtered)
    # build DataFrame and store to disk
    return PD.DataFrame(index=dt_filtered,
                        data={c1: Bx_filtered,
                              c2: By_filtered})

def process(hdf_fname,
            source_key='B_raw',
            key='B',
            he=False,
            remove_mean=True):
    """
    Process the magnetic field columns of *hdf_fname*, applying
    pre-processing (nan interpolation) and a band-pass filter. Look
    for input at *source_key* and store output under identifier
    *key*. If *he*, process the H and E magnetic field components. If
    *remove_mean*, remove the mean from each column.
    """
    logger.info('processing {}'.format(hdf_fname))
    df_raw, header = read_hdf(hdf_fname, source_key)
    dt = [PD.to_datetime(x).to_pydatetime() for x in df_raw.index.values]
    Bx_raw = df_raw['Bx'].values * 1e-9
    By_raw = df_raw['By'].values * 1e-9
    df_filtered = process_timeseries(dt,
                                     Bx_raw,
                                     By_raw,
                                     remove_mean=remove_mean)
    if he:
        Bh_raw = df_raw['Bh'].values * 1e-9
        Be_raw = df_raw['Be'].values * 1e-9
        df_he_filtered = process_timeseries(dt,
                                            Bh_raw,
                                            Be_raw,
                                            c1='Bh',
                                            c2='Be',
                                            remove_mean=remove_mean)
        df_filtered = df_filtered.join(df_he_filtered)
    write_hdf(hdf_fname, df_filtered, key, header)
    return hdf_fname

def main(argv=None):
    if argv is None:
        argv = sys.argv

    parser = ArgumentParser('Apply preprocessing steps to raw magnetometer data.',
                            formatter_class=ArgumentDefaultsHelpFormatter)
    parser.add_argument('hdf_fnames',
                        type=str,
                        nargs='*',
                        metavar='hdf_fname',
                        help='HDF file record to process')
    parser.add_argument('--source-key',
                        '-s',
                        type=str,
                        default='B_raw',
                        help='')
    parser.add_argument('--key',
                        '-k',
                        type=str,
                        default='B',
                        help='key to associate with the processed records')
    parser.add_argument('--he',
                        action='store_true',
                        help='include results in HE coordinate')
    args = parser.parse_args(argv[1:])

    for hdf_fname in args.hdf_fnames:
        process(hdf_fname,
                source_key=args.source_key,
                key=args.key,
                he=args.he)

if __name__ == '__main__':
    logging.basicConfig(level=logging.INFO)
    sys.exit(main())
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

"""Base view model."""

#------------------------------------------------------------------------------
# Imports
#------------------------------------------------------------------------------

import inspect

import numpy as np

from ...utils.array import _unique
from ...utils.selector import Selector
from ...utils._misc import _show_shortcuts
from ...utils import _as_list
from ...ext.six import string_types

#------------------------------------------------------------------------------
# Misc
#------------------------------------------------------------------------------

# Default color map for the selected clusters.
_COLORMAP = np.array([[102, 194, 165],
                      [252, 141, 98],
                      [141, 160, 203],
                      [231, 138, 195],
                      [166, 216, 84],
                      [255, 217, 47],
                      [229, 196, 148],
                      ])

def _create_view(cls, backend=None, **kwargs):
    if backend in ('pyqt4', None):
        kwargs.update({'always_on_top': True})
    return cls(**kwargs)

def _selected_clusters_colors(n_clusters):
    if n_clusters > _COLORMAP.shape[0]:
        colors = np.tile(_COLORMAP, (1 + n_clusters // _COLORMAP.shape[0], 1))
    else:
        colors = _COLORMAP
    return colors[:n_clusters, ...] / 255.

#------------------------------------------------------------------------------
# BaseViewModel
#------------------------------------------------------------------------------

class BaseViewModel(object):
    """Interface between a view and a model."""
    _view_class = None
    _view_name = ''
    _imported_params = ('position', 'size',)
    keyboard_shortcuts = {}
    scale_factor = 1.

    def __init__(self, model=None, store=None, cluster_ids=None, **kwargs):

        self._model = model
        assert store is not None
        self._store = store
        self._cluster_ids = None

        # Instanciate the underlying view.
        self._view = self._create_view(**kwargs)

        # Set passed keyword arguments as attributes.
        for param in self.imported_params():
            value = kwargs.get(param, None)
            if value is not None:
                setattr(self, param, value)

        self.on_open()
        if cluster_ids is not None:
            self.select(_as_list(cluster_ids))

    def connect(self, func):
        pass

    @classmethod
    def imported_params(cls):
        out = ()
        for base_class in inspect.getmro(cls):
            if base_class == object:
                continue
            out += base_class._imported_params
        return out

    def _create_view(self, **kwargs):
        """Create the view with the parameters passed to the constructor.

        Must be overriden."""
        return None

    @property
    def model(self):
        return self._model

    @property
    def name(self):
        return self._view_name

    @property
    def store(self):
        return self._store

    @property
    def view(self):
        return self._view

    @property
    def cluster_ids(self):
        """Selected clusters."""
        return self._cluster_ids

    @property
    def n_clusters(self):
        """Number of selected clusters."""
        return len(self._cluster_ids)

    def on_open(self):
        """Initialize the view after the model has been loaded.

        May be overriden."""

    def select(self, cluster_ids):
        """Select a set of clusters."""
        cluster_ids = _as_list(cluster_ids)
        self._cluster_ids = cluster_ids
        self.on_select(cluster_ids)

    def on_select(self, cluster_ids):
        """Update the view after a new selection has been made.

        Must be overriden."""

    def on_close(self):
        """Clear the view when the model is closed.

        May be overriden."""

    def exported_params(self, save_size_pos=True):
        """Return a dictionary of variables to save when the view is closed."""
        if save_size_pos:
            return {
                'position': self._view.position,
                'size': self._view.size,
            }
        else:
            return {}

    def show(self):
        """Show the view."""
        self._view.show()

#------------------------------------------------------------------------------
# HTMLViewModel
#------------------------------------------------------------------------------

class HTMLViewModel(BaseViewModel):
    """Widget with custom HTML code."""
    def _create_view(self, **kwargs):
        from PyQt4.QtWebKit import QWebView
        self._html = kwargs.get('html', None)
        view = QWebView()
        return view

    def on_select(self, cluster_ids):
        if isinstance(self._html, string_types):
            html = self._html
        elif inspect.isfunction(self._html):
            html = self._html(cluster_ids)
        self._view.setHtml(html)

#------------------------------------------------------------------------------
# VispyViewModel
#------------------------------------------------------------------------------

class VispyViewModel(BaseViewModel):
    """Create a VisPy view from a model.

    This object uses an internal `Selector` instance to manage spike and
    cluster selection.

    """
    _imported_params = ('n_spikes_max', 'excerpt_size')
    keyboard_shortcuts = {}
    scale_factor = 1.

    def connect(self, func):
        self._view.connect(func)

    def _create_view(self, **kwargs):
        n_spikes_max = kwargs.get('n_spikes_max', None)
        excerpt_size = kwargs.get('excerpt_size', None)
        backend = kwargs.get('backend', None)
        position = kwargs.get('position', None)
        size = kwargs.get('size', None)

        # Create the spike/cluster selector.
        self._selector = Selector(self._model.spike_clusters,
                                  n_spikes_max=n_spikes_max,
                                  excerpt_size=excerpt_size,
                                  )

        # Create the VisPy canvas.
        view = _create_view(self._view_class,
                            backend=backend,
                            position=position or (200, 200),
                            size=size or (600, 600),
                            )
        view.connect(self.on_key_press)
        return view

    @property
    def selector(self):
        return self._selector

    @property
    def cluster_ids(self):
        """Selected clusters."""
        return self._selector.selected_clusters

    @property
    def spike_ids(self):
        """Selected spikes."""
        return self._selector.selected_spikes

    @property
    def n_clusters(self):
        """Number of selected clusters."""
        return self._selector.n_clusters

    @property
    def n_spikes(self):
        """Number of selected spikes."""
        return self._selector.n_spikes

    def update_spike_clusters(self, spikes=None, spike_clusters=None):
        """Update the spike clusters and cluster colors."""
        if spikes is None:
            spikes = self.spike_ids
        if spike_clusters is None:
            spike_clusters = self.model.spike_clusters[spikes]
        n_clusters = len(_unique(spike_clusters))
        visual = self._view.visual
        # This updates the list of unique clusters in the view.
        visual.spike_clusters = spike_clusters
        visual.cluster_colors = _selected_clusters_colors(n_clusters)

    def select(self, cluster_ids):
        """Select a set of clusters."""
        self._selector.selected_clusters = cluster_ids
        self.on_select(cluster_ids)

    def on_select(self, cluster_ids):
        """Update the view after a new selection has been made.

        Must be overriden."""
        self.update_spike_clusters()
        self._view.update()

    def on_close(self):
        """Clear the view when the model is closed."""
        self._view.visual.spike_clusters = []
        self._view.update()

    def on_key_press(self, event):
        if event.key == 'h' and 'control' not in event.modifiers:
            shortcuts = self._view.keyboard_shortcuts
            shortcuts.update(self.keyboard_shortcuts)
            _show_shortcuts(shortcuts, name=self.name)
<commit_msg>Add reference to wizard in view models.<commit_after># -*- coding: utf-8 -*-

"""Base view model."""

#------------------------------------------------------------------------------
# Imports
#------------------------------------------------------------------------------

import inspect

import numpy as np

from ...utils.array import _unique
from ...utils.selector import Selector
from ...utils._misc import _show_shortcuts
from ...utils import _as_list
from ...ext.six import string_types

#------------------------------------------------------------------------------
# Misc
#------------------------------------------------------------------------------

# Default color map for the selected clusters.
_COLORMAP = np.array([[102, 194, 165],
                      [252, 141, 98],
                      [141, 160, 203],
                      [231, 138, 195],
                      [166, 216, 84],
                      [255, 217, 47],
                      [229, 196, 148],
                      ])

def _create_view(cls, backend=None, **kwargs):
    if backend in ('pyqt4', None):
        kwargs.update({'always_on_top': True})
    return cls(**kwargs)

def _selected_clusters_colors(n_clusters):
    if n_clusters > _COLORMAP.shape[0]:
        colors = np.tile(_COLORMAP, (1 + n_clusters // _COLORMAP.shape[0], 1))
    else:
        colors = _COLORMAP
    return colors[:n_clusters, ...] / 255.

#------------------------------------------------------------------------------
# BaseViewModel
#------------------------------------------------------------------------------

class BaseViewModel(object):
    """Interface between a view and a model."""
    _view_class = None
    _view_name = ''
    _imported_params = ('position', 'size',)
    keyboard_shortcuts = {}
    scale_factor = 1.

    def __init__(self, model=None, store=None, wizard=None,
                 cluster_ids=None, **kwargs):

        self._model = model
        assert store is not None
        self._store = store
        self._wizard = wizard
        self._cluster_ids = None

        # Instanciate the underlying view.
        self._view = self._create_view(**kwargs)

        # Set passed keyword arguments as attributes.
        for param in self.imported_params():
            value = kwargs.get(param, None)
            if value is not None:
                setattr(self, param, value)

        self.on_open()
        if cluster_ids is not None:
            self.select(_as_list(cluster_ids))

    def connect(self, func):
        pass

    @classmethod
    def imported_params(cls):
        out = ()
        for base_class in inspect.getmro(cls):
            if base_class == object:
                continue
            out += base_class._imported_params
        return out

    def _create_view(self, **kwargs):
        """Create the view with the parameters passed to the constructor.

        Must be overriden."""
        return None

    # Public properties
    #--------------------------------------------------------------------------

    @property
    def model(self):
        return self._model

    @property
    def name(self):
        return self._view_name

    @property
    def store(self):
        return self._store

    @property
    def view(self):
        return self._view

    @property
    def cluster_ids(self):
        """Selected clusters."""
        return self._cluster_ids

    @property
    def n_clusters(self):
        """Number of selected clusters."""
        return len(self._cluster_ids)

    # Public methods
    #--------------------------------------------------------------------------

    def select(self, cluster_ids):
        """Select a set of clusters."""
        cluster_ids = _as_list(cluster_ids)
        self._cluster_ids = cluster_ids
        self.on_select(cluster_ids)

    def exported_params(self, save_size_pos=True):
        """Return a dictionary of variables to save when the view is closed."""
        if save_size_pos:
            return {
                'position': self._view.position,
                'size': self._view.size,
            }
        else:
            return {}

    def show(self):
        """Show the view."""
        self._view.show()

    # Callback methods
    #--------------------------------------------------------------------------

    def on_open(self):
        """Initialize the view after the model has been loaded.

        May be overriden."""

    def on_select(self, cluster_ids):
        """Update the view after a new selection has been made.

        Must be overriden."""

    def on_cluster(self):
        """Called when a clustering action occurs.

        May be overriden."""

    def on_close(self):
        """Called when the model is closed.

        May be overriden."""

#------------------------------------------------------------------------------
# HTMLViewModel
#------------------------------------------------------------------------------

class HTMLViewModel(BaseViewModel):
    """Widget with custom HTML code."""

    def _create_view(self, **kwargs):
        from PyQt4.QtWebKit import QWebView
        if 'html' in kwargs:
            self._html = kwargs['html']
        view = QWebView()
        return view

    def update(self):
        if inspect.isfunction(self._html):
            html = self._html(self._cluster_ids)
        else:
            html = self._html
        self._view.setHtml(html)

    def on_select(self, cluster_ids):
        self.update()

    def on_cluster(self, up):
        self.update()

#------------------------------------------------------------------------------
# VispyViewModel
#------------------------------------------------------------------------------

class VispyViewModel(BaseViewModel):
    """Create a VisPy view from a model.

    This object uses an internal `Selector` instance to manage spike and
    cluster selection.

    """
    _imported_params = ('n_spikes_max', 'excerpt_size')
    keyboard_shortcuts = {}
    scale_factor = 1.

    def connect(self, func):
        self._view.connect(func)

    def _create_view(self, **kwargs):
        n_spikes_max = kwargs.get('n_spikes_max', None)
        excerpt_size = kwargs.get('excerpt_size', None)
        backend = kwargs.get('backend', None)
        position = kwargs.get('position', None)
        size = kwargs.get('size', None)

        # Create the spike/cluster selector.
        self._selector = Selector(self._model.spike_clusters,
                                  n_spikes_max=n_spikes_max,
                                  excerpt_size=excerpt_size,
                                  )

        # Create the VisPy canvas.
        view = _create_view(self._view_class,
                            backend=backend,
                            position=position or (200, 200),
                            size=size or (600, 600),
                            )
        view.connect(self.on_key_press)
        return view

    @property
    def selector(self):
        return self._selector

    @property
    def cluster_ids(self):
        """Selected clusters."""
        return self._selector.selected_clusters

    @property
    def spike_ids(self):
        """Selected spikes."""
        return self._selector.selected_spikes

    @property
    def n_clusters(self):
        """Number of selected clusters."""
        return self._selector.n_clusters

    @property
    def n_spikes(self):
        """Number of selected spikes."""
        return self._selector.n_spikes

    def update_spike_clusters(self, spikes=None, spike_clusters=None):
        """Update the spike clusters and cluster colors."""
        if spikes is None:
            spikes = self.spike_ids
        if spike_clusters is None:
            spike_clusters = self.model.spike_clusters[spikes]
        n_clusters = len(_unique(spike_clusters))
        visual = self._view.visual
        # This updates the list of unique clusters in the view.
        visual.spike_clusters = spike_clusters
        visual.cluster_colors = _selected_clusters_colors(n_clusters)

    def select(self, cluster_ids):
        """Select a set of clusters."""
        self._selector.selected_clusters = cluster_ids
        self.on_select(cluster_ids)

    def on_select(self, cluster_ids):
        """Update the view after a new selection has been made.

        Must be overriden."""
        self.update_spike_clusters()
        self._view.update()

    def on_close(self):
        """Clear the view when the model is closed."""
        self._view.visual.spike_clusters = []
        self._view.update()

    def on_key_press(self, event):
        if event.key == 'h' and 'control' not in event.modifiers:
            shortcuts = self._view.keyboard_shortcuts
            shortcuts.update(self.keyboard_shortcuts)
            _show_shortcuts(shortcuts, name=self.name)
<|endoftext|>


<commit_before>import re
import sys
from collections import namedtuple

from .connection import connect, Connection
from .cursor import Cursor
from .pool import create_pool, Pool

__all__ = ('connect', 'create_pool', 'Connection', 'Cursor', 'Pool',
           'version', 'version_info')

__version__ = '0.3.1'

version = __version__ + ' , Python ' + sys.version

VersionInfo = namedtuple('VersionInfo',
                         'major minor micro releaselevel serial')

def _parse_version(ver):
    RE = (r'^(?P<major>\d+)\.(?P<minor>\d+)\.'
          '(?P<micro>\d+)((?P<releaselevel>[a-z]+)(?P<serial>\d+)?)?$')
    match = re.match(RE, ver)
    try:
        major = int(match.group('major'))
        minor = int(match.group('minor'))
        micro = int(match.group('micro'))
        levels = {'rc': 'candidate',
                  'a': 'alpha',
                  'b': 'beta',
                  None: 'final'}
        releaselevel = levels[match.group('releaselevel')]
        serial = int(match.group('serial')) if match.group('serial') else 0
        return VersionInfo(major, minor, micro, releaselevel, serial)
    except Exception:
        raise ImportError("Invalid package version {}".format(ver))

version_info = _parse_version(__version__)

# make pyflakes happy
(connect, create_pool, Connection, Cursor, Pool)
<commit_msg>Bum version to 0.3.2<commit_after>import re
import sys
from collections import namedtuple

from .connection import connect, Connection
from .cursor import Cursor
from .pool import create_pool, Pool

__all__ = ('connect', 'create_pool', 'Connection', 'Cursor', 'Pool',
           'version', 'version_info')

__version__ = '0.3.2'

version = __version__ + ' , Python ' + sys.version

VersionInfo = namedtuple('VersionInfo',
                         'major minor micro releaselevel serial')

def _parse_version(ver):
    RE = (r'^(?P<major>\d+)\.(?P<minor>\d+)\.'
          '(?P<micro>\d+)((?P<releaselevel>[a-z]+)(?P<serial>\d+)?)?$')
    match = re.match(RE, ver)
    try:
        major = int(match.group('major'))
        minor = int(match.group('minor'))
        micro = int(match.group('micro'))
        levels = {'rc': 'candidate',
                  'a': 'alpha',
                  'b': 'beta',
                  None: 'final'}
        releaselevel = levels[match.group('releaselevel')]
        serial = int(match.group('serial')) if match.group('serial') else 0
        return VersionInfo(major, minor, micro, releaselevel, serial)
    except Exception:
        raise ImportError("Invalid package version {}".format(ver))

version_info = _parse_version(__version__)

# make pyflakes happy
(connect, create_pool, Connection, Cursor, Pool)
<|endoftext|>


<commit_before>"""
See test_cfsom(self) and ImagePoster for an example of how to use and call
the PlotEngine system from a simulation.

$Id$
"""
import unittest
from pprint import pprint
from topo import plot
from topo import base
from topo.bitmap import *
from topo.sheet import Sheet
from topo.inputsheet import *
from topo.simulator import *
from topo.cfsom import CFSOM
from topo.image import ImageSaver
from topo import kernelfactory
from topo.kernelfactory import GaussianFactory
from math import pi
from topo.params import Dynamic
import random
import pdb #debugger

class TestCFSom(unittest.TestCase):

    def setUp(self):
        self.s = Simulator(step_mode = 1)
        self.sheet1 = Sheet()
        self.sheet2 = Sheet()

    def test_cfsom(self):
        """
        Cut and paste of current topographica/examples/cfsom_example.py
        """
        # input generation params
        InputSheet.period = 1.0
        InputSheet.density = 900
        InputSheet.print_level = base.WARNING

        GaussianFactory.x = Dynamic(lambda : random.uniform(-0.5,0.5))
        GaussianFactory.y = Dynamic(lambda : random.uniform(-0.5,0.5))
        GaussianFactory.theta = Dynamic(lambda :random.uniform(-pi,pi))

        GaussianFactory.width = 0.02
        GaussianFactory.height = 0.9
        GaussianFactory.bounds = BoundingBox(points=((-0.8,-0.8),(0.8,0.8)))

        # cf som parameters
        CFSOM.density = 900
        CFSOM.learning_length = 10000
        CFSOM.radius_0 = 0.1

        ###########################################
        # build simulation

        base.min_print_level = base.WARNING

        s = Simulator()
        s.verbose("Creating simulation objects...")
        retina = InputSheet(input_generator=GaussianFactory())

        # Old form
        #retina = GaussianSheet(name='Retina')
        V1 = CFSOM(name='V1')
        V1.print_level = base.WARNING

        s.connect(retina,V1,delay=1)
        s.print_level = base.WARNING

        s.run(10)

suite = unittest.TestSuite()
suite.addTest(unittest.makeSuite(TestCFSom))
<commit_msg>Added test for get_projection_by_name()<commit_after>"""
See test_cfsom(self) and ImagePoster for an example of how to use and call
the PlotEngine system from a simulation.

$Id$
"""
import unittest
from pprint import pprint
from topo import plot
from topo import base
from topo.bitmap import *
from topo.sheet import Sheet
from topo.inputsheet import *
from topo.simulator import *
from topo.cfsom import CFSOM
from topo.image import ImageSaver
from topo import kernelfactory
from topo.kernelfactory import GaussianFactory
from math import pi
from topo.params import Dynamic
import random
import pdb #debugger

class TestCFSom(unittest.TestCase):

    def setUp(self):
        self.s = Simulator(step_mode = 1)
        self.sheet1 = Sheet()
        self.sheet2 = Sheet()

    def test_cfsom(self):
        """
        Cut and paste of current topographica/examples/cfsom_example.py
        """
        # input generation params
        InputSheet.period = 1.0
        InputSheet.density = 900
        InputSheet.print_level = base.WARNING

        GaussianFactory.x = Dynamic(lambda : random.uniform(-0.5,0.5))
        GaussianFactory.y = Dynamic(lambda : random.uniform(-0.5,0.5))
        GaussianFactory.theta = Dynamic(lambda :random.uniform(-pi,pi))

        GaussianFactory.width = 0.02
        GaussianFactory.height = 0.9
        GaussianFactory.bounds = BoundingBox(points=((-0.8,-0.8),(0.8,0.8)))

        # cf som parameters
        CFSOM.density = 900
        CFSOM.learning_length = 10000
        CFSOM.radius_0 = 0.1

        ###########################################
        # build simulation

        base.min_print_level = base.WARNING

        s = Simulator()
        s.verbose("Creating simulation objects...")
        retina = InputSheet(input_generator=GaussianFactory())

        # Old form
        #retina = GaussianSheet(name='Retina')
        V1 = CFSOM(name='V1')
        V1.print_level = base.WARNING

        s.connect(retina,V1,delay=1,projection_params={'name':'RtoV1'})
        s.print_level = base.WARNING

        self.assertTrue(len(V1.get_projection_by_name('RtoV1')) == 1)
        self.assertTrue(len(V1.get_projection_by_name('R1toV1')) == 0)
        s.run(10)

suite = unittest.TestSuite()
suite.addTest(unittest.makeSuite(TestCFSom))
<|endoftext|>


<commit_before>import copy
import os
import traceback
import datetime
from json import loads, load

# this folder no longer exists
# nor were the files being used
# from controls import *	#Tested

from threading import Lock
from time import time, sleep

# Import constants to initiate the hardware
from init_hw_constants import *

# Class that communicates to the i2c to pwm chip that controls the brushless motors
from hardware.motor_control import MotorControl

# Class that controls the rov movement
from movement import controller

from sensors import Pressure, IMU, OBS, ESC

from camera import Cameras

class ROV(object):

    def __init__(self, lock, data):
        self._data_lock = lock

        self._data = data
        self._new_data = False
        self._last_packet = time() - 1

        self.last_update = time()

        self._running = True

        with open("packets.json","r") as fh:
                self.dearclient = loads(load(fh))['dearclient']

        self.dearflask = {}

        self.debug = (os.environ.get("ROV_DEBUG") == "1")

        self.init_hw()

    def init_hw(self):
        self.cameras = Cameras(
            resolution='640x480',
            framerate=30,
            port=8080,
            brightness=16,
            contrast=32
        ).start()

        self.motor_control = MotorControl(
            zero_power=ZERO_POWER,
            neg_max_power=NEG_MAX_POWER,
            pos_max_power=POS_MAX_POWER,
            frequency=FREQUENCY
        )

        self.controls = controller(self.motor_control)

        self.imu = IMU()
        self.pressure = Pressure()
        self.obs = OBS()
        self.esc = ESC()

    def update(self):
        with self._data_lock:
            self.dearflask = self._data['dearflask']

        # if time() - self._last_packet > 0.5:
            # # print 'Data connection lost'
            # self.motor_control.kill()
            # self.thruster_control.stop()

        try:
	          self.imu.update()
	          self.pressure.update()
            self.obs.update()
            self.esc.update()
            df = self.dearflask
            #print df, '\n', self.dearclient, '\n\n'

        except Exception as e:
            print "Failed updating things"
            print "Exception: %s" % e
            print traceback.format_exc()
        self.dearclient['obs'] = self.obs.data
        self.dearclient['esc'] = self.esc.data

        self.dearclient['imu'] = self.imu.data
        self.dearclient['pressure'] = self.pressure.data
        self.last_update = time()

        now = datetime.datetime.now()
        self.dearclient['last_update'] = "{day}_{hour}_{minu}_{sec}_{usec}".format(day=now.day,
                                                                            hour=now.hour,
                                                                            minu=now.minute,
                                                                            sec=now.second,
                                                                            usec=now.microsecond)

        with self._data_lock:
            self._data['dearclient'] = self.dearclient

def run(lock, data):
    rov = ROV(lock, data)
    while True:
        while time() - rov.last_update < 0.01:
            sleep(0.005)

        try:
            rov.update()
        except Exception as e:
            print "Exception: %s" % e
            print traceback.format_exc()
<commit_msg>another fix, but not complete fix for integrationtest, loading packets.json into rov.<commit_after>import copy
import os
import traceback
import datetime
from json import loads, load

# this folder no longer exists
# nor were the files being used
# from controls import *	#Tested

from threading import Lock
from time import time, sleep

# Import constants to initiate the hardware
from init_hw_constants import *

# Class that communicates to the i2c to pwm chip that controls the brushless motors
from hardware.motor_control import MotorControl

# Class that controls the rov movement
from movement import controller

from sensors import Pressure, IMU, OBS, ESC

from camera import Cameras

class ROV(object):

    def __init__(self, lock, data):
        self._data_lock = lock

        self._data = data
        self._new_data = False
        self._last_packet = time() - 1

        self.last_update = time()

        self._running = True

        with open("rov/packets.json","r") as fh:
            self.dearclient = loads(load(fh))['dearclient']

        self.dearflask = {}

        self.debug = (os.environ.get("ROV_DEBUG") == "1")

        self.init_hw()

    def init_hw(self):
        self.cameras = Cameras(
            resolution='640x480',
            framerate=30,
            port=8080,
            brightness=16,
            contrast=32
        ).start()

        self.motor_control = MotorControl(
            zero_power=ZERO_POWER,
            neg_max_power=NEG_MAX_POWER,
            pos_max_power=POS_MAX_POWER,
            frequency=FREQUENCY
        )

        self.controls = controller(self.motor_control)

        self.imu = IMU()
        self.pressure = Pressure()
        self.obs = OBS()
        self.esc = ESC()

    def update(self):
        with self._data_lock:
            self.dearflask = self._data['dearflask']

        # if time() - self._last_packet > 0.5:
            # # print 'Data connection lost'
            # self.motor_control.kill()
            # self.thruster_control.stop()

        try:
            self.imu.update()
            self.pressure.update()
            self.obs.update()
            self.esc.update()
            df = self.dearflask
            #print df, '\n', self.dearclient, '\n\n'

        except Exception as e:
            print "Failed updating things"
            print "Exception: %s" % e
            print traceback.format_exc()
        self.dearclient['obs'] = self.obs.data
        self.dearclient['esc'] = self.esc.data

        self.dearclient['imu'] = self.imu.data
        self.dearclient['pressure'] = self.pressure.data
        self.last_update = time()

        now = datetime.datetime.now()
        self.dearclient['last_update'] = "{day}_{hour}_{minu}_{sec}_{usec}".format(day=now.day,
                                                                            hour=now.hour,
                                                                            minu=now.minute,
                                                                            sec=now.second,
                                                                            usec=now.microsecond)

        with self._data_lock:
            self._data['dearclient'] = self.dearclient

def run(lock, data):
    rov = ROV(lock, data)
    while True:
        while time() - rov.last_update < 0.01:
            sleep(0.005)

        try:
            rov.update()
        except Exception as e:
            print "Exception: %s" % e
            print traceback.format_exc()
<|endoftext|>


<commit_before>from __future__ import division

import sys
import logging
import warnings
from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter
from itertools import groupby

import numpy as NP
import pandas as PD
import scipy.signal

from iaga2hdf import read_hdf, write_hdf
from ..util.nan import nan_interp
from ..util.signal import lp_fir_filter
from ..stats.stats import despike

logger = logging.getLogger('pyrsss.mag.process_hdf')

"""
A NOTE ON THE COORDINATE FRAME

The following describes the XYZ convention
(from https://github.com/usgs/geomag-algorithms/blob/master/docs/algorithms/XYZ.md):

- X is the magnitude of the geographic north pole component of the H vector;
- Y is the magnitude of the east component of the H vector;
- Z is the downward component of the geomagnetic field, same as before.
"""

def minute_interval_filter_firwin():
    """
    Synthesize and return impulse response of minute interval filter
    for analysis of magnetometer data. Design a filter using the
    window method. The filter is designed to stop from 0 to 0.75 mHz
    and pass from 1 mHz to Nyquist.

    For reference: this is the filter used to process the data for the
    Space Weather 2017 paper.

    fs_1m = 1./60
    bands_1m = [0, 1e-3 - 0.25e-3, 1e-3, fs_1m/2]
    desired_1m = [0, 1]

    _ = fir_response(h_1m, bands_1m, desired_1m, Hz=fs_1m)

    FIR filter is linear phase type 1

    band 0 abs deviations from 0:
    min = 4.466e-05  (db=-87.000846)
    med = -4.218e-03
    max = 3.027e-02 (db=-30.379491

    band 1 abs deviations from 1:
    min = 4.412e-06  (db=-107.106525)
    med = 2.398e-03
    std = 4.059e-03
    max = 2.982e-02 (db=-30.509428
    """
    Ts = 60
    fs = 1. / Ts
    fn = fs / 2
    width = 0.25e-3
    cutoff = 1e-3
    ripple = 30
    numtaps, beta = scipy.signal.kaiserord(ripple, width / fn)
    if numtaps % 2 == 0:
        numtaps += 1
    return scipy.signal.firwin(numtaps,
                               cutoff - width/2,
                               width=width,
                               pass_zero=False,
                               nyq=fn)

def minute_interval_filter(N_remez=201):
    """
    Synthesize and return impulse response of minute interval filter
    for analysis of magnetometer data. Design a length *N_remez*
    min-max optimal filter. The filter is designed to stop from 0 to
    0.75 mHz and pass from 1 mHz to Nyquist.

    fs_1m = 1./60
    bands_1m = [0, 1e-3 - 0.25e-3, 1e-3, fs_1m/2]
    desired_1m = [0, 1]

    _ = fir_response(h_1m, bands_1m, desired_1m, Hz=fs_1m)

    FIR filter is linear phase type 1

    band 0 abs deviations from 0:
    min = 1.487e-05  (db=-96.553225)
    med = -1.102e-03
    max = 1.569e-03 (db=-56.089238

    band 1 abs deviations from 1:
    min = 1.018e-06  (db=-119.848775)
    med = -1.066e-05
    std = 4.820e-04
    max = 1.571e-03 (db=-56.074809
    """
    return scipy.signal.remez(N_remez,
                              [0, 0.75e-3, 1e-3, 1./60/2],
                              [0, 1],
                              [1, 1],
                              Hz=1./60)

def second_interval_filter(N_remez=4001):
    """
    Synthesize and return impulse response of second interval filter
    for analysis of magnetometer data. Design a length *N_remez*
    min-max optimal filter. The filter is designed to stop from 0 to
    0.25 mHz, pass from 1 to 100 mHz, and stop from 101 mHz to
    Nyquist.

    fs_1m = 1.
    bands_1s = [0, 1e-3 - 0.25e-3,
                1e-3, 100e-3,
                101e-3, fs_1s/2]
    desired_1s = [0, 1, 0]

    _ = fir_response(h_1m, bands_1m, desired_1m, Hz=fs_1m)

    FIR filter is linear phase type 1

    band 0: 0 Hz -- 750 \muHz
    abs deviations from 0 statistics:
    min = 6.872e-05  (db=-83.257739)
    med = -4.906e-02
    max = 7.486e-01 (db=-2.514808)

    band 1: 1 mHz -- 100 mHz
    abs deviations from 1 statistics:
    min = 2.323e-07  (db=-132.678419)
    med = -6.093e-07
    std = 5.871e-04
    max = 2.349e-03 (db=-52.584106)

    band 2: 101 mHz -- 500 mHz
    abs deviations from 0 statistics:
    min = 1.430e-07  (db=-136.893586)
    med = -1.353e-03
    max = 2.853e-03 (db=-50.895031)
    """
    return scipy.signal.remez(N_remez,
                              [0, 0.25e-3, 1e-3, 100e-3, 101e-3, 0.5],
                              [0, 1, 0],
                              [1, 1, 1],
                              Hz=1)

def consecutive_nans(x, y):
    """
    Return the maximum number of nans encountered in the logical or of
    the *x* and *y* time series.
    """
    nan = NP.isnan(x) | NP.isnan(y)
    lengths = []
    for key, group in groupby(nan):
        if key == True:
            lengths.append(len(list(group)))
    if lengths:
        return max(lengths)
    return 0

def process_timeseries(dt,
                       Bx,
                       By,
                       c1='B_X',
                       c2='B_Y',
                       despike_data=True,
                       remove_mean=True):
    """
    Process surface magnetic field measurement time series with
    indices *dt* and components *Bx* and *By*. Output a
    :class:`DataFrame` with columns *c1* and *c2* associated with the
    processed output time series. If *despike_data*, remove outliers
    prior to filtering. If *remove_mean*, remove the mean from the
    output time series.
    """
    warnings.warn('use process_df instead',
                  PendingDeprecationWarning)
    n = consecutive_nans(Bx, By)
    interval = (dt[1] - dt[0]).total_seconds()
    if n > 0:
        logger.warning('longest contiguous gap = {:.2f} minutes'.format(n * interval / 60))
    # fill data gaps via linear interpolation
    Bx = nan_interp(Bx)
    By = nan_interp(By)
    if despike_data:
        # remove outliers
        df = PD.DataFrame(index=dt,
                          data={'Bx': Bx,
                                'By': By})
        df = despike(df)
        dt = df.index.to_pydatetime()
        Bx = df.Bx.values
        By = df.By.values
    # apply 1 - 100 mHz bandpass filter
    if interval == 1.0:
        h = second_interval_filter()
    elif interval == 60.0:
        h = minute_interval_filter()
    else:
        raise ValueError('1 to 100 mHz filter not yet synthesized for {} s interval data'.format(interval))
    Bx_filtered, dt_filtered = lp_fir_filter(h, Bx, mode='valid', index=dt)
    By_filtered = lp_fir_filter(h, By, mode='valid')
    # remove mean
    if remove_mean:
        Bx_filtered -= NP.mean(Bx_filtered)
        By_filtered -= NP.mean(By_filtered)
    # build DataFrame and store to disk
    return PD.DataFrame(index=dt_filtered,
                        data={c1: Bx_filtered,
                              c2: By_filtered})

def process(hdf_fname,
            source_key='B_raw',
            key='B',
            he=False,
            despike_data=True,
            remove_mean=True):
    """
    Process the magnetic field columns of *hdf_fname*, applying
    pre-processing (nan interpolation) and a band-pass filter. Look
    for input at *source_key* and store output under identifier
    *key*. If *he*, process the H and E magnetic field components. If
    *remove_mean*, remove the mean from each column.
    """
    logger.info('processing {}'.format(hdf_fname))
    df_raw, header = read_hdf(hdf_fname, source_key)
    dt = df_raw.index.to_pydatetime()
    Bx_raw = df_raw['B_X'].values * 1e-9
    By_raw = df_raw['B_Y'].values * 1e-9
    df_filtered = process_timeseries(dt,
                                     Bx_raw,
                                     By_raw,
                                     despike_data=despike_data,
                                     remove_mean=remove_mean)
    if he:
        Bh_raw = df_raw['B_H'].values * 1e-9
        Be_raw = df_raw['B_E'].values * 1e-9
        df_he_filtered = process_timeseries(dt,
                                            Bh_raw,
                                            Be_raw,
                                            c1='B_H',
                                            c2='B_E',
                                            despike_data=despike_data,
                                            remove_mean=remove_mean)
        df_filtered = df_filtered.join(df_he_filtered)
    write_hdf(hdf_fname, df_filtered, key, header)
    return hdf_fname

def main(argv=None):
    if argv is None:
        argv = sys.argv

    parser = ArgumentParser('Apply preprocessing steps to raw magnetometer data.',
                            formatter_class=ArgumentDefaultsHelpFormatter)
    parser.add_argument('hdf_fnames',
                        type=str,
                        nargs='*',
                        metavar='hdf_fname',
                        help='HDF file record to process')
    parser.add_argument('--source-key',
                        '-s',
                        type=str,
                        default='B_raw',
                        help='')
    parser.add_argument('--key',
                        '-k',
                        type=str,
                        default='B',
                        help='key to associate with the processed records')
    parser.add_argument('--he',
                        action='store_true',
                        help='include results in HE coordinate')
    args = parser.parse_args(argv[1:])

    for hdf_fname in args.hdf_fnames:
        process(hdf_fname,
                source_key=args.source_key,
                key=args.key,
                he=args.he)

if __name__ == '__main__':
    logging.basicConfig(level=logging.INFO)
    sys.exit(main())
<commit_msg>First cut at adding new processing pipeline.<commit_after>from __future__ import division

import sys
import logging
import warnings
from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter
from datetime import timedelta
from itertools import groupby
from collections import OrderedDict

import numpy as NP
import pandas as PD
import scipy.signal

from iaga2hdf import read_hdf, write_hdf
from ..util.nan import nan_interp
from ..signal.filter import lp_fir_filter
from ..stats.stats import despike

logger = logging.getLogger('pyrsss.mag.process_hdf')

"""
A NOTE ON THE COORDINATE FRAME

The following describes the XYZ convention
(from https://github.com/usgs/geomag-algorithms/blob/master/docs/algorithms/XYZ.md):

- X is the magnitude of the geographic north pole component of the H vector;
- Y is the magnitude of the east component of the H vector;
- Z is the downward component of the geomagnetic field, same as before.
"""

def minute_interval_filter_firwin():
    """
    Synthesize and return impulse response of minute interval filter
    for analysis of magnetometer data. Design a filter using the
    window method. The filter is designed to stop from 0 to 0.75 mHz
    and pass from 1 mHz to Nyquist.

    For reference: this is the filter used to process the data for the
    Space Weather 2017 paper.

    fs_1m = 1./60
    bands_1m = [0, 1e-3 - 0.25e-3, 1e-3, fs_1m/2]
    desired_1m = [0, 1]

    _ = fir_response(h_1m, bands_1m, desired_1m, Hz=fs_1m)

    FIR filter is linear phase type 1

    band 0 abs deviations from 0:
    min = 4.466e-05  (db=-87.000846)
    med = -4.218e-03
    max = 3.027e-02 (db=-30.379491

    band 1 abs deviations from 1:
    min = 4.412e-06  (db=-107.106525)
    med = 2.398e-03
    std = 4.059e-03
    max = 2.982e-02 (db=-30.509428
    """
    Ts = 60
    fs = 1. / Ts
    fn = fs / 2
    width = 0.25e-3
    cutoff = 1e-3
    ripple = 30
    numtaps, beta = scipy.signal.kaiserord(ripple, width / fn)
    if numtaps % 2 == 0:
        numtaps += 1
    return scipy.signal.firwin(numtaps,
                               cutoff - width/2,
                               width=width,
                               pass_zero=False,
                               nyq=fn)

def minute_interval_filter(N_remez=201):
    """
    Synthesize and return impulse response of minute interval filter
    for analysis of magnetometer data. Design a length *N_remez*
    min-max optimal filter. The filter is designed to stop from 0 to
    0.75 mHz and pass from 1 mHz to Nyquist.

    fs_1m = 1./60
    bands_1m = [0, 1e-3 - 0.25e-3, 1e-3, fs_1m/2]
    desired_1m = [0, 1]

    _ = fir_response(h_1m, bands_1m, desired_1m, Hz=fs_1m)

    FIR filter is linear phase type 1

    band 0 abs deviations from 0:
    min = 1.487e-05  (db=-96.553225)
    med = -1.102e-03
    max = 1.569e-03 (db=-56.089238

    band 1 abs deviations from 1:
    min = 1.018e-06  (db=-119.848775)
    med = -1.066e-05
    std = 4.820e-04
    max = 1.571e-03 (db=-56.074809
    """
    return scipy.signal.remez(N_remez,
                              [0, 0.75e-3, 1e-3, 1./60/2],
                              [0, 1],
                              [1, 1],
                              Hz=1./60)

def second_interval_filter(N_remez=4001):
    """
    Synthesize and return impulse response of second interval filter
    for analysis of magnetometer data. Design a length *N_remez*
    min-max optimal filter. The filter is designed to stop from 0 to
    0.25 mHz, pass from 1 to 100 mHz, and stop from 101 mHz to
    Nyquist.

    fs_1m = 1.
    bands_1s = [0, 1e-3 - 0.25e-3,
                1e-3, 100e-3,
                101e-3, fs_1s/2]
    desired_1s = [0, 1, 0]

    _ = fir_response(h_1m, bands_1m, desired_1m, Hz=fs_1m)

    FIR filter is linear phase type 1

    band 0: 0 Hz -- 750 \muHz
    abs deviations from 0 statistics:
    min = 6.872e-05  (db=-83.257739)
    med = -4.906e-02
    max = 7.486e-01 (db=-2.514808)

    band 1: 1 mHz -- 100 mHz
    abs deviations from 1 statistics:
    min = 2.323e-07  (db=-132.678419)
    med = -6.093e-07
    std = 5.871e-04
    max = 2.349e-03 (db=-52.584106)

    band 2: 101 mHz -- 500 mHz
    abs deviations from 0 statistics:
    min = 1.430e-07  (db=-136.893586)
    med = -1.353e-03
    max = 2.853e-03 (db=-50.895031)
    """
    return scipy.signal.remez(N_remez,
                              [0, 0.25e-3, 1e-3, 100e-3, 101e-3, 0.5],
                              [0, 1, 0],
                              [1, 1, 1],
                              Hz=1)

def consecutive_nans(x, y):
    """
    Return the maximum number of nans encountered in the logical or of
    the *x* and *y* time series.
    """
    nan = NP.isnan(x) | NP.isnan(y)
    lengths = []
    for key, group in groupby(nan):
        if key == True:
            lengths.append(len(list(group)))
    if lengths:
        return max(lengths)
    return 0

def process_timeseries(dt,
                       Bx,
                       By,
                       c1='B_X',
                       c2='B_Y',
                       despike_data=True,
                       remove_mean=True):
    """
    Process surface magnetic field measurement time series with
    indices *dt* and components *Bx* and *By*. Output a
    :class:`DataFrame` with columns *c1* and *c2* associated with the
    processed output time series. If *despike_data*, remove outliers
    prior to filtering. If *remove_mean*, remove the mean from the
    output time series.
    """
    warnings.warn('use process_df instead',
                  PendingDeprecationWarning)
    n = consecutive_nans(Bx, By)
    interval = (dt[1] - dt[0]).total_seconds()
    if n > 0:
        logger.warning('longest contiguous gap = {:.2f} minutes'.format(n * interval / 60))
    # fill data gaps via linear interpolation
    Bx = nan_interp(Bx)
    By = nan_interp(By)
    if despike_data:
        # remove outliers
        df = PD.DataFrame(index=dt,
                          data={'Bx': Bx,
                                'By': By})
        df = despike(df)
        dt = df.index.to_pydatetime()
        Bx = df.Bx.values
        By = df.By.values
    # apply 1 - 100 mHz bandpass filter
    if interval == 1.0:
        h = second_interval_filter()
    elif interval == 60.0:
        h = minute_interval_filter()
    else:
        raise ValueError('1 to 100 mHz filter not yet synthesized for {} s interval data'.format(interval))
    Bx_filtered, dt_filtered = lp_fir_filter(h, Bx, mode='valid', index=dt)
    By_filtered = lp_fir_filter(h, By, mode='valid')
    # remove mean
    if remove_mean:
        Bx_filtered -= NP.mean(Bx_filtered)
        By_filtered -= NP.mean(By_filtered)
    # build DataFrame and store to disk
    return PD.DataFrame(index=dt_filtered,
                        data={c1: Bx_filtered,
                              c2: By_filtered})

def process(hdf_fname,
            source_key='B_raw',
            key='B',
            he=False,
            despike_data=True,
            remove_mean=True):
    """
    Process the magnetic field columns of *hdf_fname*, applying
    pre-processing (nan interpolation) and a band-pass filter. Look
    for input at *source_key* and store output under identifier
    *key*. If *he*, process the H and E magnetic field components. If
    *remove_mean*, remove the mean from each column.
    """
    logger.info('processing {}'.format(hdf_fname))
    df_raw, header = read_hdf(hdf_fname, source_key)
    dt = df_raw.index.to_pydatetime()
    Bx_raw = df_raw['B_X'].values * 1e-9
    By_raw = df_raw['B_Y'].values * 1e-9
    df_filtered = process_timeseries(dt,
                                     Bx_raw,
                                     By_raw,
                                     despike_data=despike_data,
                                     remove_mean=remove_mean)
    if he:
        Bh_raw = df_raw['B_H'].values * 1e-9
        Be_raw = df_raw['B_E'].values * 1e-9
        df_he_filtered = process_timeseries(dt,
                                            Bh_raw,
                                            Be_raw,
                                            c1='B_H',
                                            c2='B_E',
                                            despike_data=despike_data,
                                            remove_mean=remove_mean)
        df_filtered = df_filtered.join(df_he_filtered)
    write_hdf(hdf_fname, df_filtered, key, header)
    return hdf_fname

def fill_nans(df, delta=None):
    """
    """
    if not delta:
        dt_diff = NP.diff(df.index.values)
        delta_timedelta64 = min(dt_diff)
        delta_seconds = delta_timedelta64 / NP.timedelta64(1, 's')
        delta = timedelta(seconds=delta_seconds)
    logger.info('Using delta = {} (s)'.format(delta.total_seconds()))
    index_new = PD.date_range(start=df.index[0],
                              end=df.index[-1],
                              freq=delta)
    missing = sorted(set(index_new) - set(df.index))
    if missing:
        logger.warning('Missing time indices (filled by NaNs):')
        for x in missing:
            logger.warning(x)
    return df.reindex(index_new, copy=False), delta

def nan_interpolate(df):
    """

    Reference:
    https://stackoverflow.com/questions/29007830/identifying-consecutive-nans-with-pandas
    """
    sum_nan = df.isnull().sum()
    df_null_int = df.isnull().astype(int)
    for col in df.columns:
        max_run = df[col].isnull().astype(int).groupby(df[col].notnull().astype(int).cumsum()).sum()
        if sum_nan[col]:
            # BELOW IS BROKEN!!!
            pass
            # logger.warning('column {} has {} NaNs ({} max consecutive run)'.format(col,
                                                                                # sum_nan[col],
                                                                                # max_run))
    df.interpolate(inplace=True)
    return df

def process_df(df,
               delta=None,
               despike_data=True,
               subtract_median=True):
    """
    """
    if despike_data:
        logger.info('despike')
        df = despike(df)
    logger.info('Fill gaps')
    df, delta = fill_nans(df, delta=delta)
    logger.info('Gap interpolation')
    df = nan_interpolate(df)
    # apply 1 - 100 mHz bandpass filter
    interval = delta.total_seconds()
    if interval == 1.0:
        h = second_interval_filter()
    elif interval == 60.0:
        h = minute_interval_filter()
    else:
        raise ValueError('1 to 100 mHz filter not yet synthesized for {} s interval data'.format(interval))
    data = OrderedDict()
    dt = df.index.to_pydatetime()
    for i, col in enumerate(df.columns):
        logger.info('Band-pass filter {}'.format(col))
        if i == 0:
            col_filtered, dt_filtered = lp_fir_filter(h, df[col].values, mode='valid', index=dt)
        else:
            col_filtered = lp_fir_filter(h, df[col].values, mode='valid')
        data[col] = col_filtered
    df_filtered = PD.DataFrame(index=dt_filtered,
                               data=data)
    # remove median
    if subtract_median:
        logger.info('Subtract median')
        df_filtered = df_filtered.sub(df.median(axis=1), axis=0)
    return df_filtered

def process_new(hdf_fname,
                source_key='B_raw',
                key='B',
                despike_data=True,
                subtract_median=True):
    """
    """
    logger.info('processing {}'.format(hdf_fname))
    df_raw, header = read_hdf(hdf_fname, source_key)
    df = df_raw[['B_X', 'B_Y']] * 1e-9
    df_filtered = process_df(df,
                             despike_data=despike_data,
                             subtract_median=subtract_median)
    write_hdf(hdf_fname, df_filtered, key, header)
    return hdf_fname

def main(argv=None):
    if argv is None:
        argv = sys.argv

    parser = ArgumentParser('Apply preprocessing steps to raw magnetometer data.',
                            formatter_class=ArgumentDefaultsHelpFormatter)
    parser.add_argument('hdf_fnames',
                        type=str,
                        nargs='*',
                        metavar='hdf_fname',
                        help='HDF file record to process')
    parser.add_argument('--source-key',
                        '-s',
                        type=str,
                        default='B_raw',
                        help='')
    parser.add_argument('--key',
                        '-k',
                        type=str,
                        default='B',
                        help='key to associate with the processed records')
    parser.add_argument('--he',
                        action='store_true',
                        help='include results in HE coordinate')
    args = parser.parse_args(argv[1:])

    for hdf_fname in args.hdf_fnames:
        process(hdf_fname,
                source_key=args.source_key,
                key=args.key,
                he=args.he)

if __name__ == '__main__':
    logging.basicConfig(level=logging.INFO)
    sys.exit(main())
<|endoftext|>


<commit_before># coding: utf-8

from base64 import urlsafe_b64encode

def get_encoded_entry_uri(bucket: str, key=None) -> str:
    """生成七牛云API使用的EncodedEntryURI

    :param bucket: 空间名
    :param key: 文件名，默认为空

    详见：https://developer.qiniu.com/kodo/api/1276/data-format
    """
    entry_uri = "{}:{}".format(bucket, key) if key else bucket
    return urlsafe_b64encode(entry_uri.encode()).decode()
<commit_msg>添加计算etag的相关工具函数<commit_after># coding: utf-8

from io import BytesIO
from base64 import urlsafe_b64encode

import qiniu.utils

def get_encoded_entry_uri(bucket: str, key=None) -> str:
    """生成七牛云API使用的EncodedEntryURI

    :param bucket: 空间名
    :param key: 文件名，默认为空

    详见：https://developer.qiniu.com/kodo/api/1276/data-format
    """
    entry_uri = "{}:{}".format(bucket, key) if key else bucket
    return urlsafe_b64encode(entry_uri.encode()).decode()

def get_stream_etag(stream) -> str:
    """计算流数据的七牛etag

    :param stream: 流对象，文件对象或类文件对象等

    etag算法详见：https://developer.qiniu.com/kodo/manual/1231/appendix#3
    """
    return qiniu.utils.etag_stream(stream)

def get_bytes_etag(data: bytes) -> str:
    """计算字节数据的七牛etag

    :param data: 字节数据

    etag算法详见：https://developer.qiniu.com/kodo/manual/1231/appendix#3
    """
    return get_stream_etag(BytesIO(data))

def get_file_etag(filepath: str) -> str:
    """计算本地文件的七牛etag

    :param filepath: 本地文件路径

    etag算法详见：https://developer.qiniu.com/kodo/manual/1231/appendix#3
    """
    return qiniu.utils.etag(filepath)
<|endoftext|>