<commit_before>import logging, operator
import numpy as np
from bdfreader import BDFReader
from golem import DataSet, helpers
from scipy import signal

from markers import markers_to_events, biosemi_find_ghost_markers,\
  resample_markers

def sliding_window_indices(window_size, window_step, sig_len):
  '''Returns indices for a sliding window with shape [nwindows x window_size]'''
  nwindows = int(np.floor((sig_len - window_size + window_step) /
    float(window_step)))
  starts = np.arange(nwindows).reshape(nwindows, 1) * window_step
  return starts + np.arange(window_size)

def sliding_window(signal, window_size, window_step, win_func=None):
  '''Apply a sliding window to a 1D signal'''
  if signal.ndim != 1:
    raise ValueError, 'Sliding window works on 1D arrays only!'
  if win_func != None:
    if win_func.size != window_size:
      raise ValueError, 'window_size (%d) does not match win_func.size (%d)' % (
        window_size, win_func.size)
  indices = sliding_window_indices(window_size, window_step, signal.shape[0])
  windows = signal.take(indices=indices)
  if win_func != None:
    windows = windows * win_func # broadcasting matches from last dim
  return windows

def stft(signal, nfft, stepsize):
  '''Calculate the short-time Fourier transform (STFT).
  Returns [windows x FFT coefficients]'''
  wins = sliding_window(signal, nfft, stepsize, win_func=np.hanning(nfft))
  return np.fft.rfft(wins, axis=1)

def spectrogram(signal, nfft, stepsize):
  '''Calculate a spectrogram using the STFT. Returns [frames x frequencies]'''
  # abs is the *magnitude* of a complex number
  return np.abs(stft(signal, nfft, stepsize))

def popcorn(f, axis, array, *args):
  # array.shape ~ (i, j, k, l), axis = 1
  array = array.swapaxes(axis, -1)
  x_shape = array.shape[:-1]
  # x_shape~ (i, l, k)

  array = array.reshape(-1, array.shape[-1])
  # array.shape ~ (x, j)

  result = np.asarray([f(a, *args) for a in array])
  y_shape = result.shape[1:]
  # y_shape ~ (y1, y2, y3)

  result = result.reshape(x_shape + (-1,))
  # result.shape ~ (i, l, k, y)
  result = result.swapaxes(axis, -1)
  # result.shape ~ (i, y, k, l)

  final_shape = result.shape[:axis] + y_shape + result.shape[axis+1:]
  result = result.reshape(final_shape)
  # result.shape = (i, y1, y2, y3, k, l)
  return result

def bdf_dataset(fname):
  STATUS = 'Status'
  f = open(fname, 'rb')
  try:
    b = BDFReader(f)
    frames = b.read_all()
  finally:
    f.close()

  data_mask = [i for i, lab in enumerate(b.labels) if lab != STATUS]
  status_mask = b.labels.index(STATUS)
  feat_lab = [b.labels[i] for i in data_mask]
  assert min(b.sample_rate) == max(b.sample_rate)
  sample_rate = b.sample_rate[0]
  ids = (np.arange(frames.shape[0]) / float(sample_rate)).reshape(-1, 1)
  d = DataSet(
    xs=frames[:,data_mask],
    ys=frames[:,status_mask].reshape(-1, 1).astype(int) & 0xffff,
    ids=ids, feat_lab=feat_lab, cl_lab=['status'])
  ghosts = biosemi_find_ghost_markers(d.ys.flatten())
  if len(ghosts) > 0:
    loggging.getLogger('Psychic.bdf_dataset').warning(
      'Found ghost markers: %s' % str(zip(d.ys.flatten()[ghost], ghost)))
  return d

def resample_rec(d, factor, max_marker_delay=0):
  '''Resample a recording to length d.ninstances * factor'''
  new_len = int(d.ninstances * factor)
  ys = resample_markers(d.ys.flatten(), new_len,
    max_delay=max_marker_delay).reshape(-1, 1)

  # calculate xs and ids
  xs, ids = signal.resample(d.xs, new_len, t=d.ids)
  xs = xs.astype(d.xs.dtype) # keep old dtype

  # construct new DataSet
  extra = d.extra.copy()
  return DataSet(xs=xs, ys=ys, ids=ids.reshape(-1, 1), extra=extra, default=d)

def decimate_rec(d, factor, max_marker_delay=0):
  '''Decimate a recording using an anti-aliasing filter.'''
  assert isinstance(factor, int), 'Decimation factor should be an int'
  ys = resample_markers(d.ys.flatten(), d.ninstances/factor,
    max_delay=max_marker_delay).reshape(-1, 1)

  # anti-aliasing filter
  (b, a) = signal.iirfilter(8, .8 / factor, btype='lowpass', rp=0.05,
    ftype='cheby1')
  xs = d.xs.copy()
  for i in range(d.nfeatures):
    xs[:,i] = signal.filtfilt(b, a, xs[:, i])

  xs = np.ascontiguousarray(xs[::factor,:]).astype(d.xs.dtype)

  # calc ids
  ids = np.ascontiguousarray(d.ids[::factor,:]).astype(d.ids.dtype)

  # construct new DataSet
  extra = d.extra.copy()
  return DataSet(xs=xs, ys=ys, ids=ids.reshape(-1, 1), extra=extra, default=d)

def slice(d, marker_dict, offsets):
  '''
  Slice function, used to extract fixed-length snippets of EEG from a recording.
  Returns [snippet x frames x channel]
  '''
  assert len(d.feat_shape) == 1
  start_off, end_off = offsets
  xs, ys, ids = [], [], []
  (events, events_i) = markers_to_events(d.ys.flat)
  for (k, v) in marker_dict.items():
    for i in events_i[events==v]:
      (start, end) = i + start_off, i + end_off
      if start < 0 or end > d.ninstances:
        logging.getLogger('psychic.utils.slice').warning(
          'Cannot extract slice %d-%d for class %s' % (start, end, k))
        continue
      dslice = d[start:end]
      xs.append(dslice.xs)
      ys.append(v)
      ids.append(dslice.ids[0, :])

  xs = np.asarray(xs)
  feat_shape = xs.shape[1:]
  xs = xs.reshape(xs.shape[0], -1)
  event_time = dslice.ids[:, 0] - d[i].ids[0]
  time_lab = ['%.2f' % ti for ti in event_time]
  feat_nd_lab = [time_lab, d.feat_lab if d.feat_lab
    else ['f%d' % i for i in range(d.nfeatures)]]
  feat_dim_lab = ['time', 'channels']
  ys = helpers.to_one_of_n(ys)
  cl_lab = [lab for lab, _ in sorted(marker_dict.items(),
    key=operator.itemgetter(1))]
  ids = np.asarray(ids)
  d = DataSet(xs=xs, ys=ys, ids=ids, cl_lab=cl_lab,
    feat_shape=feat_shape, feat_nd_lab=feat_nd_lab,
    feat_dim_lab=feat_dim_lab, extra=dict(event_time=event_time))
  return d.sorted()

def find_segments(events, event_indices, start_mark, end_mark):
  '''Helper to find matching start/end markers in an event array'''
  events, event_indices = np.asarray(events), np.asarray(event_indices)
  assert events.size == event_indices.size
  mask = (events==start_mark) | (events==end_mark)
  sevents, sevent_ids = events[mask], event_indices[mask]
  stack, result = [], []
  for si in range(sevent_ids.size):
    if sevents[si] == start_mark:
      stack.append(sevent_ids[si])
    else:
      assert stack != [], 'Missing start marker'
      result.append((stack.pop(), sevent_ids[si]))
  if not stack == []:
    logging.getLogger('psychic.utils.find_segments').warning(
      'Did not end start marker(s) at %s' % repr(stack))
  return result

def cut_segments(d, marker_tuples, offsets=[0, 0]):
  '''
  Cut a dataset into segment using (start_marker, end_marker) tuples.
  Returns a list with DataSets.
  '''
  start_off, end_off = offsets
  segments = []
  e, ei = markers_to_events(d.ys.flat)
  for (sm, em) in marker_tuples:
    segments.extend(find_segments(e, ei, sm, em))
  segments.sort()
  return [d[s + start_off:e + end_off] for (s, e) in segments]
<commit_msg>Another logging typo :-S.<commit_after>import logging, operator
import numpy as np
from bdfreader import BDFReader
from golem import DataSet, helpers
from scipy import signal

from markers import markers_to_events, biosemi_find_ghost_markers,\
  resample_markers

def sliding_window_indices(window_size, window_step, sig_len):
  '''Returns indices for a sliding window with shape [nwindows x window_size]'''
  nwindows = int(np.floor((sig_len - window_size + window_step) /
    float(window_step)))
  starts = np.arange(nwindows).reshape(nwindows, 1) * window_step
  return starts + np.arange(window_size)

def sliding_window(signal, window_size, window_step, win_func=None):
  '''Apply a sliding window to a 1D signal'''
  if signal.ndim != 1:
    raise ValueError, 'Sliding window works on 1D arrays only!'
  if win_func != None:
    if win_func.size != window_size:
      raise ValueError, 'window_size (%d) does not match win_func.size (%d)' % (
        window_size, win_func.size)
  indices = sliding_window_indices(window_size, window_step, signal.shape[0])
  windows = signal.take(indices=indices)
  if win_func != None:
    windows = windows * win_func # broadcasting matches from last dim
  return windows

def stft(signal, nfft, stepsize):
  '''Calculate the short-time Fourier transform (STFT).
  Returns [windows x FFT coefficients]'''
  wins = sliding_window(signal, nfft, stepsize, win_func=np.hanning(nfft))
  return np.fft.rfft(wins, axis=1)

def spectrogram(signal, nfft, stepsize):
  '''Calculate a spectrogram using the STFT. Returns [frames x frequencies]'''
  # abs is the *magnitude* of a complex number
  return np.abs(stft(signal, nfft, stepsize))

def popcorn(f, axis, array, *args):
  # array.shape ~ (i, j, k, l), axis = 1
  array = array.swapaxes(axis, -1)
  x_shape = array.shape[:-1]
  # x_shape~ (i, l, k)

  array = array.reshape(-1, array.shape[-1])
  # array.shape ~ (x, j)

  result = np.asarray([f(a, *args) for a in array])
  y_shape = result.shape[1:]
  # y_shape ~ (y1, y2, y3)

  result = result.reshape(x_shape + (-1,))
  # result.shape ~ (i, l, k, y)
  result = result.swapaxes(axis, -1)
  # result.shape ~ (i, y, k, l)

  final_shape = result.shape[:axis] + y_shape + result.shape[axis+1:]
  result = result.reshape(final_shape)
  # result.shape = (i, y1, y2, y3, k, l)
  return result

def bdf_dataset(fname):
  STATUS = 'Status'
  f = open(fname, 'rb')
  try:
    b = BDFReader(f)
    frames = b.read_all()
  finally:
    f.close()

  data_mask = [i for i, lab in enumerate(b.labels) if lab != STATUS]
  status_mask = b.labels.index(STATUS)
  feat_lab = [b.labels[i] for i in data_mask]
  assert min(b.sample_rate) == max(b.sample_rate)
  sample_rate = b.sample_rate[0]
  ids = (np.arange(frames.shape[0]) / float(sample_rate)).reshape(-1, 1)
  d = DataSet(
    xs=frames[:,data_mask],
    ys=frames[:,status_mask].reshape(-1, 1).astype(int) & 0xffff,
    ids=ids, feat_lab=feat_lab, cl_lab=['status'])
  ghosts = biosemi_find_ghost_markers(d.ys.flatten())
  if len(ghosts) > 0:
    logging.getLogger('Psychic.bdf_dataset').warning(
      'Found ghost markers: %s' % str(zip(d.ys.flatten()[ghost], ghost)))
  return d

def resample_rec(d, factor, max_marker_delay=0):
  '''Resample a recording to length d.ninstances * factor'''
  new_len = int(d.ninstances * factor)
  ys = resample_markers(d.ys.flatten(), new_len,
    max_delay=max_marker_delay).reshape(-1, 1)

  # calculate xs and ids
  xs, ids = signal.resample(d.xs, new_len, t=d.ids)
  xs = xs.astype(d.xs.dtype) # keep old dtype

  # construct new DataSet
  extra = d.extra.copy()
  return DataSet(xs=xs, ys=ys, ids=ids.reshape(-1, 1), extra=extra, default=d)

def decimate_rec(d, factor, max_marker_delay=0):
  '''Decimate a recording using an anti-aliasing filter.'''
  assert isinstance(factor, int), 'Decimation factor should be an int'
  ys = resample_markers(d.ys.flatten(), d.ninstances/factor,
    max_delay=max_marker_delay).reshape(-1, 1)

  # anti-aliasing filter
  (b, a) = signal.iirfilter(8, .8 / factor, btype='lowpass', rp=0.05,
    ftype='cheby1')
  xs = d.xs.copy()
  for i in range(d.nfeatures):
    xs[:,i] = signal.filtfilt(b, a, xs[:, i])

  xs = np.ascontiguousarray(xs[::factor,:]).astype(d.xs.dtype)

  # calc ids
  ids = np.ascontiguousarray(d.ids[::factor,:]).astype(d.ids.dtype)

  # construct new DataSet
  extra = d.extra.copy()
  return DataSet(xs=xs, ys=ys, ids=ids.reshape(-1, 1), extra=extra, default=d)

def slice(d, marker_dict, offsets):
  '''
  Slice function, used to extract fixed-length snippets of EEG from a recording.
  Returns [snippet x frames x channel]
  '''
  assert len(d.feat_shape) == 1
  start_off, end_off = offsets
  xs, ys, ids = [], [], []
  (events, events_i) = markers_to_events(d.ys.flat)
  for (k, v) in marker_dict.items():
    for i in events_i[events==v]:
      (start, end) = i + start_off, i + end_off
      if start < 0 or end > d.ninstances:
        logging.getLogger('psychic.utils.slice').warning(
          'Cannot extract slice %d-%d for class %s' % (start, end, k))
        continue
      dslice = d[start:end]
      xs.append(dslice.xs)
      ys.append(v)
      ids.append(dslice.ids[0, :])

  xs = np.asarray(xs)
  feat_shape = xs.shape[1:]
  xs = xs.reshape(xs.shape[0], -1)
  event_time = dslice.ids[:, 0] - d[i].ids[0]
  time_lab = ['%.2f' % ti for ti in event_time]
  feat_nd_lab = [time_lab, d.feat_lab if d.feat_lab
    else ['f%d' % i for i in range(d.nfeatures)]]
  feat_dim_lab = ['time', 'channels']
  ys = helpers.to_one_of_n(ys)
  cl_lab = [lab for lab, _ in sorted(marker_dict.items(),
    key=operator.itemgetter(1))]
  ids = np.asarray(ids)
  d = DataSet(xs=xs, ys=ys, ids=ids, cl_lab=cl_lab,
    feat_shape=feat_shape, feat_nd_lab=feat_nd_lab,
    feat_dim_lab=feat_dim_lab, extra=dict(event_time=event_time))
  return d.sorted()

def find_segments(events, event_indices, start_mark, end_mark):
  '''Helper to find matching start/end markers in an event array'''
  events, event_indices = np.asarray(events), np.asarray(event_indices)
  assert events.size == event_indices.size
  mask = (events==start_mark) | (events==end_mark)
  sevents, sevent_ids = events[mask], event_indices[mask]
  stack, result = [], []
  for si in range(sevent_ids.size):
    if sevents[si] == start_mark:
      stack.append(sevent_ids[si])
    else:
      assert stack != [], 'Missing start marker'
      result.append((stack.pop(), sevent_ids[si]))
  if not stack == []:
    logging.getLogger('psychic.utils.find_segments').warning(
      'Did not end start marker(s) at %s' % repr(stack))
  return result

def cut_segments(d, marker_tuples, offsets=[0, 0]):
  '''
  Cut a dataset into segment using (start_marker, end_marker) tuples.
  Returns a list with DataSets.
  '''
  start_off, end_off = offsets
  segments = []
  e, ei = markers_to_events(d.ys.flat)
  for (sm, em) in marker_tuples:
    segments.extend(find_segments(e, ei, sm, em))
  segments.sort()
  return [d[s + start_off:e + end_off] for (s, e) in segments]
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# Copyright 2015 Objectif Libre
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.
#
import decimal

from gnocchiclient import client as gclient
from keystoneauth1 import loading as ks_loading
from oslo_config import cfg
from oslo_utils import units

from cloudkitty import collector
from cloudkitty import utils as ck_utils

GNOCCHI_COLLECTOR_OPTS = 'gnocchi_collector'
gnocchi_collector_opts = ks_loading.get_auth_common_conf_options()

cfg.CONF.register_opts(gnocchi_collector_opts, GNOCCHI_COLLECTOR_OPTS)
ks_loading.register_session_conf_options(
    cfg.CONF,
    GNOCCHI_COLLECTOR_OPTS)
ks_loading.register_auth_conf_options(
    cfg.CONF,
    GNOCCHI_COLLECTOR_OPTS)
CONF = cfg.CONF

class GnocchiCollector(collector.BaseCollector):
    collector_name = 'gnocchi'
    dependencies = ('GnocchiTransformer',
                    'CloudKittyFormatTransformer')
    retrieve_mappings = {
        'compute': 'instance',
        'image': 'image',
        'volume': 'volume',
        'network.bw.out': 'instance_network_interface',
        'network.bw.in': 'instance_network_interface',
        'network.floating': 'network',
    }
    metrics_mappings = {
        'compute': [
            ('vcpus', 'max'),
            ('memory', 'max'),
            ('cpu', 'max'),
            ('disk.root.size', 'max'),
            ('disk.ephemeral.size', 'max')],
        'image': [
            ('image.size', 'max'),
            ('image.download', 'max'),
            ('image.serve', 'max')],
        'volume': [
            ('volume.size', 'max')],
        'network.bw.out': [
            ('network.outgoing.bytes', 'max')],
        'network.bw.in': [
            ('network.incoming.bytes', 'max')],
        'network.floating': [
            ('ip.floating', 'max')],
    }
    units_mappings = {
        'compute': (1, 'instance'),
        'image': ('image.size', 'MB'),
        'volume': ('volume.size', 'GB'),
        'network.bw.out': ('network.outgoing.bytes', 'MB'),
        'network.bw.in': ('network.incoming.bytes', 'MB'),
        'network.floating': (1, 'ip'),
    }
    default_unit = (1, 'unknown')

    def __init__(self, transformers, **kwargs):
        super(GnocchiCollector, self).__init__(transformers, **kwargs)

        self.t_gnocchi = self.transformers['GnocchiTransformer']
        self.t_cloudkitty = self.transformers['CloudKittyFormatTransformer']

        self.auth = ks_loading.load_auth_from_conf_options(
            CONF,
            GNOCCHI_COLLECTOR_OPTS)
        self.session = ks_loading.load_session_from_conf_options(
            CONF,
            GNOCCHI_COLLECTOR_OPTS,
            auth=self.auth)
        self._conn = gclient.Client(
            '1',
            session=self.session)

    @classmethod
    def get_metadata(cls, resource_name, transformers):
        info = super(GnocchiCollector, cls).get_metadata(resource_name,
                                                         transformers)
        try:
            info["metadata"].extend(transformers['GnocchiTransformer']
                                    .get_metadata(resource_name))
            info["unit"] = cls.units_mappings[resource_name][1]
        except KeyError:
            pass
        return info

    @classmethod
    def gen_filter(cls, cop='=', lop='and', **kwargs):
        """Generate gnocchi filter from kwargs.

        :param cop: Comparison operator.
        :param lop: Logical operator in case of multiple filters.
        """
        q_filter = []
        for kwarg in sorted(kwargs):
            q_filter.append({cop: {kwarg: kwargs[kwarg]}})
        if len(kwargs) > 1:
            return cls.extend_filter(q_filter, lop=lop)
        else:
            return q_filter[0] if len(kwargs) else {}

    @classmethod
    def extend_filter(cls, *args, **kwargs):
        """Extend an existing gnocchi filter with multiple operations.

        :param lop: Logical operator in case of multiple filters.
        """
        lop = kwargs.get('lop', 'and')
        filter_list = []
        for cur_filter in args:
            if isinstance(cur_filter, dict) and cur_filter:
                filter_list.append(cur_filter)
            elif isinstance(cur_filter, list):
                filter_list.extend(cur_filter)
        if len(filter_list) > 1:
            return {lop: filter_list}
        else:
            return filter_list[0] if len(filter_list) else {}

    def _generate_time_filter(self, start, end):
        """Generate timeframe filter.

        :param start: Start of the timeframe.
        :param end: End of the timeframe if needed.
        """
        time_filter = list()
        time_filter.append(self.extend_filter(
            self.gen_filter(ended_at=None),
            self.gen_filter(cop=">=", ended_at=start),
            lop='or'))
        time_filter.append(
            self.gen_filter(cop="<=", started_at=end))
        return time_filter

    def _expand_metrics(self, resources, mappings, start, end):
        for resource in resources:
            metrics = resource.get('metrics', {})
            for name, aggregate in mappings:
                try:
                    values = self._conn.metric.get_measures(
                        metric=metrics[name],
                        start=ck_utils.ts2dt(start),
                        stop=ck_utils.ts2dt(end),
                        aggregation=aggregate)
                    # NOTE(sheeprine): Get the list of values for the current
                    # metric and get the first result value.
                    # [point_date, granularity, value]
                    # ["2015-11-24T00:00:00+00:00", 86400.0, 64.0]
                    resource[name] = values[0][2]
                except IndexError:
                    resource[name] = None
                except KeyError:
                    # Skip metrics not found
                    pass

    def get_resources(self, resource_name, start, end,
                      project_id, q_filter=None):
        """Get resources during the timeframe.

        :param resource_name: Resource name to filter on.
        :type resource_name: str
        :param start: Start of the timeframe.
        :param end: End of the timeframe if needed.
        :param project_id: Filter on a specific tenant/project.
        :type project_id: str
        :param q_filter: Append a custom filter.
        :type q_filter: list
        """
        # NOTE(sheeprine): We first get the list of every resource running
        # without any details or history.
        # Then we get information about the resource getting details and
        # history.

        # Translating the resource name if needed
        query_parameters = self._generate_time_filter(start, end)
        resource_type = self.retrieve_mappings.get(resource_name)
        query_parameters.append(
            self.gen_filter(cop="=", type=resource_type))
        query_parameters.append(
            self.gen_filter(project_id=project_id))
        if q_filter:
            query_parameters.append(q_filter)
        resources = self._conn.resource.search(
            resource_type=resource_type,
            query=self.extend_filter(*query_parameters))
        return resources

    def resource_info(self, resource_name, start, end, project_id,
                      q_filter=None):
        qty, unit = self.units_mappings.get(resource_name, self.default_unit)
        resources = self.get_resources(resource_name, start, end,
                                       project_id=project_id,
                                       q_filter=q_filter)
        formated_resources = list()
        for resource in resources:
            resource_data = self.t_gnocchi.strip_resource_data(
                resource_name, resource)
            mappings = self.metrics_mappings[resource_name]
            self._expand_metrics([resource_data], mappings, start, end)
            resource_data.pop('metrics', None)
            # Convert network.bw.in, network.bw.out and image unit to MB
            if resource.get('type') == 'instance_network_interface':
                resource_data[qty] = (
                    decimal.Decimal(resource_data[qty]) / units.M)
            elif resource.get('type') == 'image':
                resource_data[qty] = (
                    decimal.Decimal(resource_data[qty]) / units.Mi)
            data = self.t_cloudkitty.format_item(
                resource_data, unit,
                decimal.Decimal(
                    qty if isinstance(qty, int) else resource_data[qty]))
            # NOTE(sheeprine): Reference to gnocchi resource used by storage
            data['resource_id'] = data['desc']['resource_id']
            formated_resources.append(data)
        return formated_resources

    def retrieve(self, resource_name, start, end, project_id, q_filter=None):
        resources = self.resource_info(resource_name, start, end,
                                       project_id=project_id,
                                       q_filter=q_filter)
        if not resources:
            raise collector.NoDataCollected(self.collector_name, resource_name)
        return self.t_cloudkitty.format_service(resource_name, resources)
<commit_msg>Fix gnocchi metric collection<commit_after># -*- coding: utf-8 -*-
# Copyright 2015 Objectif Libre
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.
#
import decimal

from gnocchiclient import client as gclient
from keystoneauth1 import loading as ks_loading
from oslo_config import cfg
from oslo_utils import units

from cloudkitty import collector
from cloudkitty import utils as ck_utils

GNOCCHI_COLLECTOR_OPTS = 'gnocchi_collector'
gnocchi_collector_opts = ks_loading.get_auth_common_conf_options()

cfg.CONF.register_opts(gnocchi_collector_opts, GNOCCHI_COLLECTOR_OPTS)
ks_loading.register_session_conf_options(
    cfg.CONF,
    GNOCCHI_COLLECTOR_OPTS)
ks_loading.register_auth_conf_options(
    cfg.CONF,
    GNOCCHI_COLLECTOR_OPTS)
CONF = cfg.CONF

class GnocchiCollector(collector.BaseCollector):
    collector_name = 'gnocchi'
    dependencies = ('GnocchiTransformer',
                    'CloudKittyFormatTransformer')
    retrieve_mappings = {
        'compute': 'instance',
        'image': 'image',
        'volume': 'volume',
        'network.bw.out': 'instance_network_interface',
        'network.bw.in': 'instance_network_interface',
        'network.floating': 'network',
    }
    metrics_mappings = {
        'compute': [
            ('vcpus', 'max'),
            ('memory', 'max'),
            ('cpu', 'max'),
            ('disk.root.size', 'max'),
            ('disk.ephemeral.size', 'max')],
        'image': [
            ('image.size', 'max'),
            ('image.download', 'max'),
            ('image.serve', 'max')],
        'volume': [
            ('volume.size', 'max')],
        'network.bw.out': [
            ('network.outgoing.bytes', 'max')],
        'network.bw.in': [
            ('network.incoming.bytes', 'max')],
        'network.floating': [
            ('ip.floating', 'max')],
    }
    units_mappings = {
        'compute': (1, 'instance'),
        'image': ('image.size', 'MB'),
        'volume': ('volume.size', 'GB'),
        'network.bw.out': ('network.outgoing.bytes', 'MB'),
        'network.bw.in': ('network.incoming.bytes', 'MB'),
        'network.floating': (1, 'ip'),
    }
    default_unit = (1, 'unknown')

    def __init__(self, transformers, **kwargs):
        super(GnocchiCollector, self).__init__(transformers, **kwargs)

        self.t_gnocchi = self.transformers['GnocchiTransformer']
        self.t_cloudkitty = self.transformers['CloudKittyFormatTransformer']

        self.auth = ks_loading.load_auth_from_conf_options(
            CONF,
            GNOCCHI_COLLECTOR_OPTS)
        self.session = ks_loading.load_session_from_conf_options(
            CONF,
            GNOCCHI_COLLECTOR_OPTS,
            auth=self.auth)
        self._conn = gclient.Client(
            '1',
            session=self.session)

    @classmethod
    def get_metadata(cls, resource_name, transformers):
        info = super(GnocchiCollector, cls).get_metadata(resource_name,
                                                         transformers)
        try:
            info["metadata"].extend(transformers['GnocchiTransformer']
                                    .get_metadata(resource_name))
            info["unit"] = cls.units_mappings[resource_name][1]
        except KeyError:
            pass
        return info

    @classmethod
    def gen_filter(cls, cop='=', lop='and', **kwargs):
        """Generate gnocchi filter from kwargs.

        :param cop: Comparison operator.
        :param lop: Logical operator in case of multiple filters.
        """
        q_filter = []
        for kwarg in sorted(kwargs):
            q_filter.append({cop: {kwarg: kwargs[kwarg]}})
        if len(kwargs) > 1:
            return cls.extend_filter(q_filter, lop=lop)
        else:
            return q_filter[0] if len(kwargs) else {}

    @classmethod
    def extend_filter(cls, *args, **kwargs):
        """Extend an existing gnocchi filter with multiple operations.

        :param lop: Logical operator in case of multiple filters.
        """
        lop = kwargs.get('lop', 'and')
        filter_list = []
        for cur_filter in args:
            if isinstance(cur_filter, dict) and cur_filter:
                filter_list.append(cur_filter)
            elif isinstance(cur_filter, list):
                filter_list.extend(cur_filter)
        if len(filter_list) > 1:
            return {lop: filter_list}
        else:
            return filter_list[0] if len(filter_list) else {}

    def _generate_time_filter(self, start, end):
        """Generate timeframe filter.

        :param start: Start of the timeframe.
        :param end: End of the timeframe if needed.
        """
        time_filter = list()
        time_filter.append(self.extend_filter(
            self.gen_filter(ended_at=None),
            self.gen_filter(cop=">=", ended_at=start),
            lop='or'))
        time_filter.append(
            self.gen_filter(cop="<=", started_at=end))
        return time_filter

    def _expand_metrics(self, resources, mappings, start, end):
        for resource in resources:
            metrics = resource.get('metrics', {})
            for name, aggregate in mappings:
                try:
                    values = self._conn.metric.get_measures(
                        metric=metrics[name],
                        start=ck_utils.ts2dt(start),
                        stop=ck_utils.ts2dt(end),
                        aggregation=aggregate)
                    # NOTE(sheeprine): Get the list of values for the current
                    # metric and get the first result value.
                    # [point_date, granularity, value]
                    # ["2015-11-24T00:00:00+00:00", 86400.0, 64.0]
                    resource[name] = values[0][2]
                except IndexError:
                    resource[name] = 0
                except KeyError:
                    # Skip metrics not found
                    pass

    def get_resources(self, resource_name, start, end,
                      project_id, q_filter=None):
        """Get resources during the timeframe.

        :param resource_name: Resource name to filter on.
        :type resource_name: str
        :param start: Start of the timeframe.
        :param end: End of the timeframe if needed.
        :param project_id: Filter on a specific tenant/project.
        :type project_id: str
        :param q_filter: Append a custom filter.
        :type q_filter: list
        """
        # NOTE(sheeprine): We first get the list of every resource running
        # without any details or history.
        # Then we get information about the resource getting details and
        # history.

        # Translating the resource name if needed
        query_parameters = self._generate_time_filter(start, end)
        resource_type = self.retrieve_mappings.get(resource_name)
        query_parameters.append(
            self.gen_filter(cop="=", type=resource_type))
        query_parameters.append(
            self.gen_filter(project_id=project_id))
        if q_filter:
            query_parameters.append(q_filter)
        resources = self._conn.resource.search(
            resource_type=resource_type,
            query=self.extend_filter(*query_parameters))
        return resources

    def resource_info(self, resource_name, start, end, project_id,
                      q_filter=None):
        qty, unit = self.units_mappings.get(resource_name, self.default_unit)
        resources = self.get_resources(resource_name, start, end,
                                       project_id=project_id,
                                       q_filter=q_filter)
        formated_resources = list()
        for resource in resources:
            resource_data = self.t_gnocchi.strip_resource_data(
                resource_name, resource)
            mappings = self.metrics_mappings[resource_name]
            self._expand_metrics([resource_data], mappings, start, end)
            resource_data.pop('metrics', None)
            # Convert network.bw.in, network.bw.out and image unit to MB
            if resource.get('type') == 'instance_network_interface':
                resource_data[qty] = (
                    decimal.Decimal(resource_data[qty]) / units.M)
            elif resource.get('type') == 'image':
                resource_data[qty] = (
                    decimal.Decimal(resource_data[qty]) / units.Mi)
            data = self.t_cloudkitty.format_item(
                resource_data, unit,
                decimal.Decimal(
                    qty if isinstance(qty, int) else resource_data[qty]))
            # NOTE(sheeprine): Reference to gnocchi resource used by storage
            data['resource_id'] = data['desc']['resource_id']
            formated_resources.append(data)
        return formated_resources

    def retrieve(self, resource_name, start, end, project_id, q_filter=None):
        resources = self.resource_info(resource_name, start, end,
                                       project_id=project_id,
                                       q_filter=q_filter)
        if not resources:
            raise collector.NoDataCollected(self.collector_name, resource_name)
        return self.t_cloudkitty.format_service(resource_name, resources)
<|endoftext|>


<commit_before># Copyright 2013 Red Hat, Inc.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

"""Provides threading and stack-trace models

This module defines classes representing thread, green
thread, and stack trace data models
"""

import traceback

import openstack.common.report.models.with_default_views as mwdv
import openstack.common.report.views.text.threading as text_views

class StackTraceModel(mwdv.ModelWithDefaultViews):
    """A Stack Trace Model

    This model holds data from a python stack trace,
    commonly extracted from running thread information

    :param stack_state: the python stack_state object
    """

    def __init__(self, stack_state):
        super(StackTraceModel, self).__init__(
            text_view=text_views.StackTraceView())

        if (stack_state is not None):
            self['lines'] = [
                {'filename': fn, 'line': ln, 'name': nm, 'code': cd}
                for fn, ln, nm, cd in traceback.extract_stack(stack_state)
            ]

            if stack_state.f_exc_type is not None:
                self['root_exception'] = {
                    'type': stack_state.f_exc_type,
                    'value': stack_state.f_exc_value
                }
            else:
                self['root_exception'] = None
        else:
            self['lines'] = []
            self['root_exception'] = None

class ThreadModel(mwdv.ModelWithDefaultViews):
    """A Thread Model

    This model holds data for information about an
    individual thread.  It holds both a thread id,
    as well as a stack trace for the thread

    .. seealso::

        Class :class:`StackTraceModel`

    :param int thread_id: the id of the thread
    :param stack: the python stack state for the current thread
    """

    # threadId, stack in sys._current_frams().items()
    def __init__(self, thread_id, stack):
        super(ThreadModel, self).__init__(text_view=text_views.ThreadView())

        self['thread_id'] = thread_id
        self['stack_trace'] = StackTraceModel(stack)

class GreenThreadModel(mwdv.ModelWithDefaultViews):
    """A Green Thread Model

    This model holds data for information about an
    individual thread.  Unlike the thread model,
    it holds just a stack trace, since green threads
    do not have thread ids.

    .. seealso::

        Class :class:`StackTraceModel`

    :param stack: the python stack state for the green thread
    """

    # gr in greenpool.coroutines_running  --> gr.gr_frame
    def __init__(self, stack):
        super(GreenThreadModel, self).__init__(
            {'stack_trace': StackTraceModel(stack)},
            text_view=text_views.GreenThreadView())
<commit_msg>Fix tests.unit.reports.test_guru_meditation_report under Python3<commit_after># Copyright 2013 Red Hat, Inc.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

"""Provides threading and stack-trace models

This module defines classes representing thread, green
thread, and stack trace data models
"""

import traceback

import openstack.common.report.models.with_default_views as mwdv
import openstack.common.report.views.text.threading as text_views

class StackTraceModel(mwdv.ModelWithDefaultViews):
    """A Stack Trace Model

    This model holds data from a python stack trace,
    commonly extracted from running thread information

    :param stack_state: the python stack_state object
    """

    def __init__(self, stack_state):
        super(StackTraceModel, self).__init__(
            text_view=text_views.StackTraceView())

        if (stack_state is not None):
            self['lines'] = [
                {'filename': fn, 'line': ln, 'name': nm, 'code': cd}
                for fn, ln, nm, cd in traceback.extract_stack(stack_state)
            ]
            # FIXME(flepied): under Python3 f_exc_type doesn't exist
            # anymore so we lose information about exceptions
            if getattr(stack_state, 'f_exc_type', None) is not None:
                self['root_exception'] = {
                    'type': stack_state.f_exc_type,
                    'value': stack_state.f_exc_value}
            else:
                self['root_exception'] = None
        else:
            self['lines'] = []
            self['root_exception'] = None

class ThreadModel(mwdv.ModelWithDefaultViews):
    """A Thread Model

    This model holds data for information about an
    individual thread.  It holds both a thread id,
    as well as a stack trace for the thread

    .. seealso::

        Class :class:`StackTraceModel`

    :param int thread_id: the id of the thread
    :param stack: the python stack state for the current thread
    """

    # threadId, stack in sys._current_frams().items()
    def __init__(self, thread_id, stack):
        super(ThreadModel, self).__init__(text_view=text_views.ThreadView())

        self['thread_id'] = thread_id
        self['stack_trace'] = StackTraceModel(stack)

class GreenThreadModel(mwdv.ModelWithDefaultViews):
    """A Green Thread Model

    This model holds data for information about an
    individual thread.  Unlike the thread model,
    it holds just a stack trace, since green threads
    do not have thread ids.

    .. seealso::

        Class :class:`StackTraceModel`

    :param stack: the python stack state for the green thread
    """

    # gr in greenpool.coroutines_running  --> gr.gr_frame
    def __init__(self, stack):
        super(GreenThreadModel, self).__init__(
            {'stack_trace': StackTraceModel(stack)},
            text_view=text_views.GreenThreadView())
<|endoftext|>


<commit_before>import logging
import sys
from . import config

__author__ = 'karl.gong'

class PConsole:
    def __init__(self, out):
        self.out = out

    def write(self, msg):
        self.out.write(msg)

    def write_line(self, msg):
        self.write(msg + "\n")

pconsole = PConsole(sys.stdout)

class PReporter:
    def __init__(self):
        pass

    def debug(self, msg):
        self.__log(logging.DEBUG, msg)

    def info(self, msg):
        self.__log(logging.INFO, msg)

    def warn(self, msg):
        self.__log(logging.WARN, msg)

    def error(self, msg):
        self.__log(logging.ERROR, msg)

    def critical(self, msg):
        self.__log(logging.CRITICAL, msg)

    def __log(self, level, msg):
        from . import testexecutor

        running_test_fixture = testexecutor.get_property("running_test_fixture")
        running_test_fixture.logs.append((logging.getLevelName(level).lower(), str(msg),))

        if config.get_option("verbose"):
            # output to pconsole
            message = "[%s] %s" % (running_test_fixture.full_name, msg)
            pconsole.write_line(message)

preporter = PReporter()
<commit_msg>Update logs.append<commit_after>import logging
import sys

from . import config

__author__ = 'karl.gong'

class PConsole:
    def __init__(self, out):
        self.out = out

    def write(self, msg):
        self.out.write(msg)

    def write_line(self, msg):
        self.write(msg + "\n")

pconsole = PConsole(sys.stdout)

class PReporter:
    def __init__(self):
        pass

    def debug(self, msg):
        self.__log(logging.DEBUG, msg)

    def info(self, msg):
        self.__log(logging.INFO, msg)

    def warn(self, msg):
        self.__log(logging.WARN, msg)

    def error(self, msg):
        self.__log(logging.ERROR, msg)

    def critical(self, msg):
        self.__log(logging.CRITICAL, msg)

    def __log(self, level, msg):
        from . import testexecutor

        running_test_fixture = testexecutor.get_property("running_test_fixture")
        running_test_fixture.logs.append({"level": logging.getLevelName(level).lower(), "message": str(msg)})

        if config.get_option("verbose"):
            # output to pconsole
            message = "[%s] %s" % (running_test_fixture.full_name, msg)
            pconsole.write_line(message)

preporter = PReporter()
<|endoftext|>


<commit_before>from time import time

try:
    from ipykernel.kernelapp import IPKernelApp
    from ipywidgets import widgets, IntProgress, HBox, HTML, VBox
    from IPython.display import clear_output, display
    from ipywidgets.widgets.interaction import show_inline_matplotlib_plots
    import matplotlib.pyplot as plt
    IN_NOTEBOOK = IPKernelApp.initialized()
except: IN_NOTEBOOK = False

__all__ = ['master_bar', 'progress_bar']

def format_time(t):
    t = int(t)
    h,m,s = t//3600, (t//60)%60, t%60
    if h!= 0: return f'{h}:{m:02d}:{s:02d}'
    else:     return f'{m:02d}:{s:02d}'

class ProgressBar():
    update_every = 0.2

    def __init__(self, gen, display=True, leave=True, parent=None):
        self._gen,self.total = gen,len(gen)
        if parent is None: self.leave,self.display = leave,display
        else:
            self.leave,self.display=False,False
            parent.add_child(self)
        self.comment = ''

    def on_iter_begin(self): pass
    def on_interrupt(self): pass
    def on_iter_end(self): pass
    def on_update(self, val, text): pass

    def __iter__(self):
        self.on_iter_begin()
        self.update(0)
        try:
            for i,o in enumerate(self._gen):
                yield o
                self.update(i+1)
        except:
            self.on_interrupt()
            raise
        self.on_iter_end()

    def update(self, val):
        if val == 0:
            self.start_t = self.last_t = time()
            self.pred_t = 0
            self.last_v,self.wait_for = 0,1
            self.update_bar(0)
        elif val >= self.last_v + self.wait_for:
            cur_t = time()
            avg_t = (cur_t - self.start_t) / val
            self.wait_for = max(int(self.update_every / avg_t),1)
            self.pred_t = avg_t * self.total
            self.last_v,self.last_t = val,cur_t
            self.update_bar(val)
        elif val == self.total - 1:
            self.last_v, self.last_t = val, time()
            self.pred_t = self.last_t - self.start_t
            self.update_bar(self.total)

    def update_bar(self, val):
        elapsed_t = self.last_t - self.start_t
        remaining_t = format_time(self.pred_t - elapsed_t)
        elapsed_t = format_time(elapsed_t)
        end = '' if len(self.comment) == 0 else f' {self.comment}'
        self.on_update(val, f'{100 * val/self.total:.2f}% [{val}/{self.total} {elapsed_t}<{remaining_t}{end}]')

class MasterBar():
    def __init__(self, gen, cls): self.first_bar = cls(gen, display=False)

    def __iter__(self):
        self.on_iter_begin()
        for o in self.first_bar: yield o
        self.on_iter_end()

    def on_iter_begin(self): pass
    def on_iter_end(self): pass
    def add_child(self, child): pass
    def write(self, line):      pass
    def update_graph(self, graphs, x_bounds, y_bounds): pass

class NBProgressBar(ProgressBar):
    def __init__(self,gen, display=True, leave=True, parent=None):
        self.progress,self.text = IntProgress(min=0, max=len(gen)), HTML()
        self.box = HBox([self.progress, self.text])
        super().__init__(gen, display, leave, parent)

    def on_iter_begin(self):
        if self.display: display(self.box)
        self.is_active=True

    def on_interrupt(self):
        self.progress.bar_style = 'danger'
        self.is_active=False

    def on_iter_end(self):
        if not self.leave: self.box.close()
        self.is_active=False

    def on_update(self, val, text):
        self.text.value = text
        self.progress.value = val

class NBMasterBar(MasterBar):
    names = ['train', 'valid']
    def __init__(self, gen, hide_graph=False, order=None):
        super().__init__(gen, NBProgressBar)
        self.text = HTML()
        self.vbox = VBox([self.first_bar.box, self.text])
        if order is None: order = ['pb1', 'text', 'pb2', 'graph']
        self.inner_dict = {'pb1':self.first_bar.box, 'text':self.text}
        self.hide_graph,self.order = hide_graph,order

    def on_iter_begin(self): display(self.vbox)
    def on_iter_end(self):
        if hasattr(self, 'fig'): self.fig.clear()

    def add_child(self, child):
        self.child = child
        self.inner_dict['pb2'] = self.child.box
        if hasattr(self,'out'): self.show(['pb1', 'pb2', 'text', 'graph'])
        else:                   self.show(['pb1', 'pb2', 'text'])

    def show(self, child_names):
        to_show = [name for name in self.order if name in child_names]
        self.vbox.children = [self.inner_dict[n] for n in to_show]

    def write(self, line): self.text.value += line + '<p>'

    def update_graph(self, graphs, x_bounds=None, y_bounds=None):
        if self.hide_graph: return
        self.out = widgets.Output()
        if not hasattr(self, 'fig'):
            self.fig, self.ax = plt.subplots(1, figsize=(6,4))
        self.out = widgets.Output()
        self.inner_dict['graph'] = self.out
        self.ax.clear()
        if len(self.names) < len(graphs): self.names += [''] * (len(graphs) - len(self.names))
        for g,n in zip(graphs,self.names): self.ax.plot(*g, label=n)
        self.ax.legend(loc='upper right')
        if x_bounds is not None: self.ax.set_xlim(*x_bounds)
        if y_bounds is not None: self.ax.set_ylim(*y_bounds)
        with self.out:
            clear_output(wait=True)
            display(self.ax.figure)
        if hasattr(self,'child') and self.child.is_active: self.show(['pb1', 'pb2', 'text', 'graph'])
        else: self.show(['pb1', 'text', 'graph'])

class ConsoleProgressBar(ProgressBar):
    length:int=50
    fill:str='█'

    def __init__(self,gen, display=True, leave=True, parent=None):
        self.max_len,self.prefix = 0,''
        super().__init__(gen, display, leave, parent)

    def on_iter_end(self):
        if not self.leave:
            print(f'\r{self.prefix}' + ' ' * (self.max_len - len(f'\r{self.prefix}')), end = '\r')

    def on_update(self, val, text):
        if self.display:
            filled_len = int(self.length * val // self.total)
            bar = self.fill * filled_len + '-' * (self.length - filled_len)
            to_write = f'\r{self.prefix} |{bar}| {text}'
            if len(to_write) > self.max_len: self.max_len=len(to_write)
            print(to_write, end = '\r')

class ConsoleMasterBar(MasterBar):
    def __init__(self, gen, hide_graph=False, order=None): super().__init__(gen, ConsoleProgressBar)

    def add_child(self, child):
        self.child = child
        self.child.prefix = f'Epoch {self.first_bar.last_v+1}/{self.first_bar.total} :'
        self.child.display = True

    def write(self, line): print(line)

if IN_NOTEBOOK: master_bar, progress_bar = NBMasterBar, NBProgressBar
else:           master_bar, progress_bar = ConsoleMasterBar, ConsoleProgressBar

<commit_msg>Refactors<commit_after>from time import time

try:
    from ipykernel.kernelapp import IPKernelApp
    from ipywidgets import widgets, IntProgress, HBox, HTML, VBox
    from IPython.display import clear_output, display
    from ipywidgets.widgets.interaction import show_inline_matplotlib_plots
    import matplotlib.pyplot as plt
    IN_NOTEBOOK = IPKernelApp.initialized()
except: IN_NOTEBOOK = False

__all__ = ['master_bar', 'progress_bar']

def format_time(t):
    t = int(t)
    h,m,s = t//3600, (t//60)%60, t%60
    if h!= 0: return f'{h}:{m:02d}:{s:02d}'
    else:     return f'{m:02d}:{s:02d}'

class ProgressBar():
    update_every = 0.2

    def __init__(self, gen, display=True, leave=True, parent=None):
        self._gen,self.total = gen,len(gen)
        if parent is None: self.leave,self.display = leave,display
        else:
            self.leave,self.display=False,False
            parent.add_child(self)
        self.comment = ''

    def on_iter_begin(self): pass
    def on_interrupt(self): pass
    def on_iter_end(self): pass
    def on_update(self, val, text): pass

    def __iter__(self):
        self.on_iter_begin()
        self.update(0)
        try:
            for i,o in enumerate(self._gen):
                yield o
                self.update(i+1)
        except:
            self.on_interrupt()
            raise
        self.on_iter_end()

    def update(self, val):
        if val == 0:
            self.start_t = self.last_t = time()
            self.pred_t = 0
            self.last_v,self.wait_for = 0,1
            self.update_bar(0)
        elif val >= self.last_v + self.wait_for or val == self.total:
            cur_t = time()
            avg_t = (cur_t - self.start_t) / val
            self.wait_for = max(int(self.update_every / avg_t),1)
            self.pred_t = avg_t * self.total
            self.last_v,self.last_t = val,cur_t
            self.update_bar(val)

    def update_bar(self, val):
        elapsed_t = self.last_t - self.start_t
        remaining_t = format_time(self.pred_t - elapsed_t)
        elapsed_t = format_time(elapsed_t)
        end = '' if len(self.comment) == 0 else f' {self.comment}'
        self.on_update(val, f'{100 * val/self.total:.2f}% [{val}/{self.total} {elapsed_t}<{remaining_t}{end}]')

class MasterBar():
    def __init__(self, gen, cls): self.first_bar = cls(gen, display=False)

    def __iter__(self):
        self.on_iter_begin()
        for o in self.first_bar: yield o
        self.on_iter_end()

    def on_iter_begin(self): pass
    def on_iter_end(self): pass
    def add_child(self, child): pass
    def write(self, line):      pass
    def update_graph(self, graphs, x_bounds, y_bounds): pass

class NBProgressBar(ProgressBar):
    def __init__(self,gen, display=True, leave=True, parent=None):
        self.progress,self.text = IntProgress(min=0, max=len(gen)), HTML()
        self.box = HBox([self.progress, self.text])
        super().__init__(gen, display, leave, parent)

    def on_iter_begin(self):
        if self.display: display(self.box)
        self.is_active=True

    def on_interrupt(self):
        self.progress.bar_style = 'danger'
        self.is_active=False

    def on_iter_end(self):
        if not self.leave: self.box.close()
        self.is_active=False

    def on_update(self, val, text):
        self.text.value = text
        self.progress.value = val

class NBMasterBar(MasterBar):
    names = ['train', 'valid']
    def __init__(self, gen, hide_graph=False, order=None):
        super().__init__(gen, NBProgressBar)
        self.text = HTML()
        self.vbox = VBox([self.first_bar.box, self.text])
        if order is None: order = ['pb1', 'text', 'pb2', 'graph']
        self.inner_dict = {'pb1':self.first_bar.box, 'text':self.text}
        self.hide_graph,self.order = hide_graph,order

    def on_iter_begin(self): display(self.vbox)
    def on_iter_end(self):
        if hasattr(self, 'fig'): self.fig.clear()

    def add_child(self, child):
        self.child = child
        self.inner_dict['pb2'] = self.child.box
        if hasattr(self,'out'): self.show(['pb1', 'pb2', 'text', 'graph'])
        else:                   self.show(['pb1', 'pb2', 'text'])

    def show(self, child_names):
        to_show = [name for name in self.order if name in child_names]
        self.vbox.children = [self.inner_dict[n] for n in to_show]

    def write(self, line): self.text.value += line + '<p>'

    def update_graph(self, graphs, x_bounds=None, y_bounds=None):
        if self.hide_graph: return
        self.out = widgets.Output()
        if not hasattr(self, 'fig'):
            self.fig, self.ax = plt.subplots(1, figsize=(6,4))
        self.out = widgets.Output()
        self.inner_dict['graph'] = self.out
        self.ax.clear()
        if len(self.names) < len(graphs): self.names += [''] * (len(graphs) - len(self.names))
        for g,n in zip(graphs,self.names): self.ax.plot(*g, label=n)
        self.ax.legend(loc='upper right')
        if x_bounds is not None: self.ax.set_xlim(*x_bounds)
        if y_bounds is not None: self.ax.set_ylim(*y_bounds)
        with self.out:
            clear_output(wait=True)
            display(self.ax.figure)
        if hasattr(self,'child') and self.child.is_active: self.show(['pb1', 'pb2', 'text', 'graph'])
        else: self.show(['pb1', 'text', 'graph'])

class ConsoleProgressBar(ProgressBar):
    length:int=50
    fill:str='█'

    def __init__(self,gen, display=True, leave=True, parent=None):
        self.max_len,self.prefix = 0,''
        super().__init__(gen, display, leave, parent)

    def on_iter_end(self):
        if not self.leave:
            print(f'\r{self.prefix}' + ' ' * (self.max_len - len(f'\r{self.prefix}')), end = '\r')

    def on_update(self, val, text):
        if self.display:
            filled_len = int(self.length * val // self.total)
            bar = self.fill * filled_len + '-' * (self.length - filled_len)
            to_write = f'\r{self.prefix} |{bar}| {text}'
            if len(to_write) > self.max_len: self.max_len=len(to_write)
            print(to_write, end = '\r')

class ConsoleMasterBar(MasterBar):
    def __init__(self, gen, hide_graph=False, order=None): super().__init__(gen, ConsoleProgressBar)

    def add_child(self, child):
        self.child = child
        self.child.prefix = f'Epoch {self.first_bar.last_v+1}/{self.first_bar.total} :'
        self.child.display = True

    def write(self, line): print(line)

if IN_NOTEBOOK: master_bar, progress_bar = NBMasterBar, NBProgressBar
else:           master_bar, progress_bar = ConsoleMasterBar, ConsoleProgressBar

<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# Copyright 2017 Objectif Libre
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.
#
# @author: Luka Peschke
#
from keystoneauth1 import loading as ks_loading
from keystoneclient.v3 import client as ks_client
from monascaclient import client as mclient
from oslo_config import cfg
from oslo_log import log as logging
from voluptuous import All
from voluptuous import In
from voluptuous import Length
from voluptuous import Required
from voluptuous import Schema

from cloudkitty import collector
from cloudkitty import transformer
from cloudkitty import utils as ck_utils

LOG = logging.getLogger(__name__)

MONASCA_API_VERSION = '2_0'
COLLECTOR_MONASCA_OPTS = 'collector_monasca'
collector_monasca_opts = ks_loading.get_auth_common_conf_options()

cfg.CONF.register_opts(collector_monasca_opts, COLLECTOR_MONASCA_OPTS)
ks_loading.register_session_conf_options(
    cfg.CONF,
    COLLECTOR_MONASCA_OPTS)
ks_loading.register_auth_conf_options(
    cfg.CONF,
    COLLECTOR_MONASCA_OPTS)
CONF = cfg.CONF

METRICS_CONF = ck_utils.load_conf(CONF.collect.metrics_conf)

MONASCA_EXTRA_SCHEMA = {
    Required('extra_args'): {
        # Key corresponding to the resource id in a metric's dimensions
        # Allows to adapt the resource identifier. Should not need to be
        # modified in a standard OpenStack installation
        Required('resource_key', default='resource_id'):
            All(str, Length(min=1)),
        # This is needed to allow filtering on the project for the Openstack
        # usecase. May be removed in following releases
        Required('scope_key', default='project_id'): All(str, Length(min=1)),
        Required('aggregation_method', default='max'):
            In(['max', 'mean', 'min']),
    },
}

class EndpointNotFound(Exception):
    """Exception raised if the Monasca endpoint is not found"""
    pass

class MonascaCollector(collector.BaseCollector):
    collector_name = 'monasca'

    @staticmethod
    def check_configuration(conf):
        """Check metrics configuration

        """
        conf = Schema(collector.CONF_BASE_SCHEMA)(conf)
        metric_schema = Schema(collector.METRIC_BASE_SCHEMA).extend(
            MONASCA_EXTRA_SCHEMA)

        output = dict()
        for metric_name, metric in conf['metrics'].items():
            output[metric_name] = metric_schema(metric)
        return output

    def __init__(self, transformers, **kwargs):
        super(MonascaCollector, self).__init__(transformers, **kwargs)

        self.auth = ks_loading.load_auth_from_conf_options(
            CONF,
            COLLECTOR_MONASCA_OPTS)
        self.session = ks_loading.load_session_from_conf_options(
            CONF,
            COLLECTOR_MONASCA_OPTS,
            auth=self.auth)
        self.ks_client = ks_client.Client(session=self.session)
        self.mon_endpoint = self._get_monasca_endpoint()
        if not self.mon_endpoint:
            raise EndpointNotFound()
        self._conn = mclient.Client(
            api_version=MONASCA_API_VERSION,
            session=self.session,
            endpoint=self.mon_endpoint)

    # NOTE(lukapeschke) This function should be removed as soon as the endpoint
    # it no longer required by monascaclient
    def _get_monasca_endpoint(self, service_name='monasca',
                              endpoint_interface_type='public'):
        service_list = self.ks_client.services.list(name=service_name)
        if not service_list:
            return None
        mon_service = service_list[0]
        endpoints = self.ks_client.endpoints.list(mon_service.id)
        for endpoint in endpoints:
            if endpoint.interface == endpoint_interface_type:
                return endpoint.url
        return None

    def _get_metadata(self, metric_name, transformers, conf):
        info = {}
        info['unit'] = conf['metrics'][metric_name]['unit']

        dimension_names = self._conn.metric.list_dimension_names(
            metric_name=metric_name)
        info['metadata'] = [d['dimension_name'] for d in dimension_names]
        return info

    # NOTE(lukapeschke) if anyone sees a better way to do this,
    # please make a patch
    @classmethod
    def get_metadata(cls, resource_type, transformers, conf):
        args = {
            'transformers': transformer.get_transformers(),
            'period': conf['period']}
        tmp = cls(**args)
        return tmp._get_metadata(resource_type, transformers, conf)

    def _get_dimensions(self, metric_name, project_id, q_filter):
        extra_args = self.conf[metric_name]['extra_args']
        dimensions = {}
        if project_id:
            dimensions[extra_args['scope_key']] = project_id
        if q_filter:
            dimensions.update(q_filter)
        return dimensions

    def _fetch_measures(self, metric_name, start, end,
                        project_id=None, q_filter=None):
        """Get measures for given metric during the timeframe.

        :param metric_name: metric name to filter on.
        :type metric_name: str
        :param start: Start of the timeframe.
        :param end: End of the timeframe if needed.
        :param project_id: Filter on a specific tenant/project.
        :type project_id: str
        :param q_filter: Append a custom filter.
        :type q_filter: list
        """

        dimensions = self._get_dimensions(metric_name, project_id, q_filter)
        group_by = self.conf[metric_name]['groupby']
        # NOTE(lpeschke): One aggregated measure per collect period
        period = end - start

        extra_args = self.conf[metric_name]['extra_args']
        return self._conn.metrics.list_statistics(
            name=metric_name,
            merge_metrics=True,
            dimensions=dimensions,
            start_time=ck_utils.ts2dt(start),
            end_time=ck_utils.ts2dt(end),
            period=period,
            statistics=extra_args['aggregation_method'],
            group_by=group_by)

    def _fetch_metrics(self, metric_name, start, end,
                       project_id=None, q_filter=None):
        """List active metrics during the timeframe.

        :param metric_name: metric name to filter on.
        :type metric_name: str
        :param start: Start of the timeframe.
        :param end: End of the timeframe if needed.
        :param project_id: Filter on a specific tenant/project.
        :type project_id: str
        :param q_filter: Append a custom filter.
        :type q_filter: list
        """
        dimensions = self._get_dimensions(metric_name, project_id, q_filter)
        metrics = self._conn.metrics.list(
            name=metric_name,
            dimensions=dimensions,
            start_time=ck_utils.ts2dt(start),
            end_time=ck_utils.ts2dt(end),
        )

        resource_key = self.conf[metric_name]['extra_args']['resource_key']

        return {metric['dimensions'][resource_key]:
                metric['dimensions'] for metric in metrics}

    def _format_data(self, metconf, data, resources_info=None):
        """Formats Monasca data to CK data.

        Returns metadata, groupby and qty

        """
        groupby = data['dimensions']

        resource_key = metconf['extra_args']['resource_key']
        metadata = dict()
        if resources_info:
            resource = resources_info[groupby[resource_key]]
            for i in metconf['metadata']:
                metadata[i] = resource.get(i, '')

        qty = data['statistics'][0][1]
        converted_qty = ck_utils.convert_unit(
            qty, metconf['factor'], metconf['offset'])
        mutated_qty = ck_utils.mutate(converted_qty, metconf['mutate'])
        return metadata, groupby, mutated_qty

    def fetch_all(self, metric_name, start, end,
                  project_id=None, q_filter=None):
        met = self.conf[metric_name]

        data = self._fetch_measures(
            metric_name,
            start,
            end,
            project_id=project_id,
            q_filter=q_filter,
        )

        resources_info = None
        if met['metadata']:
            resources_info = self._fetch_metrics(
                metric_name,
                start,
                end,
                project_id=project_id,
                q_filter=q_filter,
            )

        formated_resources = list()
        for d in data:
            if len(d['statistics']):
                metadata, groupby, qty = self._format_data(
                    met, d, resources_info)
                data = self.t_cloudkitty.format_item(
                    groupby,
                    metadata,
                    met['unit'],
                    qty=qty,
                )
                formated_resources.append(data)
        return formated_resources
<commit_msg>Ensure resource_key is in groupby in Monasca collector<commit_after># -*- coding: utf-8 -*-
# Copyright 2017 Objectif Libre
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.
#
# @author: Luka Peschke
#
from keystoneauth1 import loading as ks_loading
from keystoneclient.v3 import client as ks_client
from monascaclient import client as mclient
from oslo_config import cfg
from oslo_log import log as logging
from voluptuous import All
from voluptuous import In
from voluptuous import Length
from voluptuous import Required
from voluptuous import Schema

from cloudkitty import collector
from cloudkitty import transformer
from cloudkitty import utils as ck_utils

LOG = logging.getLogger(__name__)

MONASCA_API_VERSION = '2_0'
COLLECTOR_MONASCA_OPTS = 'collector_monasca'
collector_monasca_opts = ks_loading.get_auth_common_conf_options()

cfg.CONF.register_opts(collector_monasca_opts, COLLECTOR_MONASCA_OPTS)
ks_loading.register_session_conf_options(
    cfg.CONF,
    COLLECTOR_MONASCA_OPTS)
ks_loading.register_auth_conf_options(
    cfg.CONF,
    COLLECTOR_MONASCA_OPTS)
CONF = cfg.CONF

METRICS_CONF = ck_utils.load_conf(CONF.collect.metrics_conf)

MONASCA_EXTRA_SCHEMA = {
    Required('extra_args'): {
        # Key corresponding to the resource id in a metric's dimensions
        # Allows to adapt the resource identifier. Should not need to be
        # modified in a standard OpenStack installation
        Required('resource_key', default='resource_id'):
            All(str, Length(min=1)),
        # This is needed to allow filtering on the project for the Openstack
        # usecase. May be removed in following releases
        Required('scope_key', default='project_id'): All(str, Length(min=1)),
        Required('aggregation_method', default='max'):
            In(['max', 'mean', 'min']),
    },
}

class EndpointNotFound(Exception):
    """Exception raised if the Monasca endpoint is not found"""
    pass

class MonascaCollector(collector.BaseCollector):
    collector_name = 'monasca'

    @staticmethod
    def check_configuration(conf):
        """Check metrics configuration

        """
        conf = Schema(collector.CONF_BASE_SCHEMA)(conf)
        metric_schema = Schema(collector.METRIC_BASE_SCHEMA).extend(
            MONASCA_EXTRA_SCHEMA)

        output = dict()
        for metric_name, metric in conf['metrics'].items():
            output[metric_name] = metric_schema(metric)
        return output

    def __init__(self, transformers, **kwargs):
        super(MonascaCollector, self).__init__(transformers, **kwargs)

        self.auth = ks_loading.load_auth_from_conf_options(
            CONF,
            COLLECTOR_MONASCA_OPTS)
        self.session = ks_loading.load_session_from_conf_options(
            CONF,
            COLLECTOR_MONASCA_OPTS,
            auth=self.auth)
        self.ks_client = ks_client.Client(session=self.session)
        self.mon_endpoint = self._get_monasca_endpoint()
        if not self.mon_endpoint:
            raise EndpointNotFound()
        self._conn = mclient.Client(
            api_version=MONASCA_API_VERSION,
            session=self.session,
            endpoint=self.mon_endpoint)

    # NOTE(lukapeschke) This function should be removed as soon as the endpoint
    # it no longer required by monascaclient
    def _get_monasca_endpoint(self, service_name='monasca',
                              endpoint_interface_type='public'):
        service_list = self.ks_client.services.list(name=service_name)
        if not service_list:
            return None
        mon_service = service_list[0]
        endpoints = self.ks_client.endpoints.list(mon_service.id)
        for endpoint in endpoints:
            if endpoint.interface == endpoint_interface_type:
                return endpoint.url
        return None

    def _get_metadata(self, metric_name, transformers, conf):
        info = {}
        info['unit'] = conf['metrics'][metric_name]['unit']

        dimension_names = self._conn.metric.list_dimension_names(
            metric_name=metric_name)
        info['metadata'] = [d['dimension_name'] for d in dimension_names]
        return info

    # NOTE(lukapeschke) if anyone sees a better way to do this,
    # please make a patch
    @classmethod
    def get_metadata(cls, resource_type, transformers, conf):
        args = {
            'transformers': transformer.get_transformers(),
            'period': conf['period']}
        tmp = cls(**args)
        return tmp._get_metadata(resource_type, transformers, conf)

    def _get_dimensions(self, metric_name, project_id, q_filter):
        extra_args = self.conf[metric_name]['extra_args']
        dimensions = {}
        if project_id:
            dimensions[extra_args['scope_key']] = project_id
        if q_filter:
            dimensions.update(q_filter)
        return dimensions

    def _fetch_measures(self, metric_name, start, end,
                        project_id=None, q_filter=None):
        """Get measures for given metric during the timeframe.

        :param metric_name: metric name to filter on.
        :type metric_name: str
        :param start: Start of the timeframe.
        :param end: End of the timeframe if needed.
        :param project_id: Filter on a specific tenant/project.
        :type project_id: str
        :param q_filter: Append a custom filter.
        :type q_filter: list
        """

        dimensions = self._get_dimensions(metric_name, project_id, q_filter)
        group_by = self.conf[metric_name]['groupby']
        resource_key = self.conf[metric_name]['extra_args']['resource_key']
        if resource_key not in group_by:
            LOG.error('Resource key "{}" is not in group_by keys: "{}". '
                      'Please adapt your configuration.'.format(
                          resource_key, group_by))
            raise collector.NoDataCollected(self.collector_name, metric_name)

        # NOTE(lpeschke): One aggregated measure per collect period
        period = end - start

        extra_args = self.conf[metric_name]['extra_args']
        return self._conn.metrics.list_statistics(
            name=metric_name,
            merge_metrics=True,
            dimensions=dimensions,
            start_time=ck_utils.ts2dt(start),
            end_time=ck_utils.ts2dt(end),
            period=period,
            statistics=extra_args['aggregation_method'],
            group_by=group_by)

    def _fetch_metrics(self, metric_name, start, end,
                       project_id=None, q_filter=None):
        """List active metrics during the timeframe.

        :param metric_name: metric name to filter on.
        :type metric_name: str
        :param start: Start of the timeframe.
        :param end: End of the timeframe if needed.
        :param project_id: Filter on a specific tenant/project.
        :type project_id: str
        :param q_filter: Append a custom filter.
        :type q_filter: list
        """
        dimensions = self._get_dimensions(metric_name, project_id, q_filter)
        metrics = self._conn.metrics.list(
            name=metric_name,
            dimensions=dimensions,
            start_time=ck_utils.ts2dt(start),
            end_time=ck_utils.ts2dt(end),
        )

        resource_key = self.conf[metric_name]['extra_args']['resource_key']

        return {metric['dimensions'][resource_key]:
                metric['dimensions'] for metric in metrics}

    def _format_data(self, metconf, data, resources_info=None):
        """Formats Monasca data to CK data.

        Returns metadata, groupby and qty

        """
        groupby = data['dimensions']

        resource_key = metconf['extra_args']['resource_key']
        metadata = dict()
        if resources_info:
            resource = resources_info[groupby[resource_key]]
            for i in metconf['metadata']:
                metadata[i] = resource.get(i, '')

        qty = data['statistics'][0][1]
        converted_qty = ck_utils.convert_unit(
            qty, metconf['factor'], metconf['offset'])
        mutated_qty = ck_utils.mutate(converted_qty, metconf['mutate'])
        return metadata, groupby, mutated_qty

    def fetch_all(self, metric_name, start, end,
                  project_id=None, q_filter=None):
        met = self.conf[metric_name]

        data = self._fetch_measures(
            metric_name,
            start,
            end,
            project_id=project_id,
            q_filter=q_filter,
        )

        resources_info = None
        if met['metadata']:
            resources_info = self._fetch_metrics(
                metric_name,
                start,
                end,
                project_id=project_id,
                q_filter=q_filter,
            )

        formated_resources = list()
        for d in data:
            if len(d['statistics']):
                metadata, groupby, qty = self._format_data(
                    met, d, resources_info)
                data = self.t_cloudkitty.format_item(
                    groupby,
                    metadata,
                    met['unit'],
                    qty=qty,
                )
                formated_resources.append(data)
        return formated_resources
<|endoftext|>


<commit_before>r"""
Subsampling (:mod:`skbio.math.subsample`)
=========================================

.. currentmodule:: skbio.math.subsample

This module provides functionality for subsampling from vectors of counts.

Functions
---------

.. autosummary::
   :toctree: generated/

   subsample

"""

# ----------------------------------------------------------------------------
# Copyright (c) 2013--, scikit-bio development team.
#
# Distributed under the terms of the Modified BSD License.
#
# The full license is in the file COPYING.txt, distributed with this software.
# ----------------------------------------------------------------------------

from __future__ import division

from warnings import warn

import numpy as np

from skbio.core.warning import EfficiencyWarning
try:
    from _subsample import _subsample_without_replacement
except ImportError:
    pass

def subsample(counts, n, replace=False):
    """Randomly subsample from a vector of counts, with or without replacement.

    Parameters
    ----------
    counts : 1-D array_like
        Vector of counts (integers) to randomly subsample from.
    n : int
        Number of items to subsample from `counts`. Must be less than or equal
        to the sum of `counts`.
    replace : bool, optional
        If ``True``, subsample with replacement. If ``False`` (the default),
        subsample without replacement.

    Returns
    -------
    subsampled : ndarray
        Subsampled vector of counts where the sum of the elements equals `n`
        (i.e., ``subsampled.sum() == n``). Will have the same shape as
        `counts`.

    Raises
    ------
    TypeError
        If `counts` cannot be safely converted to an integer datatype.
    ValueError
        If `n` is less than zero or greater than the sum of `counts`.

    Raises
    ------
    EfficiencyWarning
        If the accelerated code isn't present or hasn't been compiled.

    Notes
    -----
    If subsampling is performed without replacement (``replace=False``), a copy
    of `counts` is returned if `n` is equal to the number of items in `counts`,
    as all items will be chosen from the original vector.

    If subsampling is performed with replacement (``replace=True``) and `n` is
    equal to the number of items in `counts`, the subsampled vector that is
    returned may not necessarily be the same vector as `counts`.

    Examples
    --------
    Subsample 4 items (without replacement) from a vector of counts:

    >>> import numpy as np
    >>> from skbio.math.subsample import subsample
    >>> a = np.array([4, 5, 0, 2, 1])
    >>> sub = subsample(a, 4)
    >>> sub.sum()
    4
    >>> sub.shape
    (5,)

    Trying to subsample an equal number of items (without replacement) results
    in the same vector as our input:

    >>> subsample([0, 3, 0, 1], 4)
    array([0, 3, 0, 1])

    Subsample 5 items (with replacement):

    >>> sub = subsample([1, 0, 1, 2, 2, 3, 0, 1], 5, replace=True)
    >>> sub.sum()
    5
    >>> sub.shape
    (8,)

    """
    if n < 0:
        raise ValueError("n cannot be negative.")

    counts = np.asarray(counts)
    counts = counts.astype(int, casting='safe')

    if counts.ndim != 1:
        raise ValueError("Only 1-D vectors are supported.")

    counts_sum = counts.sum()
    if n > counts_sum:
        raise ValueError("Cannot subsample more items than exist in input "
                         "counts vector.")

    if replace:
        probs = counts / counts_sum
        result = np.random.multinomial(n, probs)
    else:
        if counts_sum == n:
            result = counts
        else:
            try:
                result = _subsample_without_replacement(counts, n, counts_sum)
            except NameError:
                warn("Accelerated subsampling without replacement isn't"
                     " available.", EfficiencyWarning)

                nz = counts.nonzero()[0]
                unpacked = np.concatenate([np.repeat(np.array(i,), counts[i])
                                           for i in nz])
                permuted = np.random.permutation(unpacked)[:n]

                result = np.zeros(len(counts), dtype=int)
                for p in permuted:
                    result[p] += 1

    return result
<commit_msg>BUG: There was a implicit relative import waiting to bite<commit_after>r"""
Subsampling (:mod:`skbio.math.subsample`)
=========================================

.. currentmodule:: skbio.math.subsample

This module provides functionality for subsampling from vectors of counts.

Functions
---------

.. autosummary::
   :toctree: generated/

   subsample

"""

# ----------------------------------------------------------------------------
# Copyright (c) 2013--, scikit-bio development team.
#
# Distributed under the terms of the Modified BSD License.
#
# The full license is in the file COPYING.txt, distributed with this software.
# ----------------------------------------------------------------------------

from __future__ import absolute_import, division, print_function

from warnings import warn

import numpy as np

from skbio.core.warning import EfficiencyWarning
try:
    from ._subsample import _subsample_without_replacement
except ImportError:
    pass

def subsample(counts, n, replace=False):
    """Randomly subsample from a vector of counts, with or without replacement.

    Parameters
    ----------
    counts : 1-D array_like
        Vector of counts (integers) to randomly subsample from.
    n : int
        Number of items to subsample from `counts`. Must be less than or equal
        to the sum of `counts`.
    replace : bool, optional
        If ``True``, subsample with replacement. If ``False`` (the default),
        subsample without replacement.

    Returns
    -------
    subsampled : ndarray
        Subsampled vector of counts where the sum of the elements equals `n`
        (i.e., ``subsampled.sum() == n``). Will have the same shape as
        `counts`.

    Raises
    ------
    TypeError
        If `counts` cannot be safely converted to an integer datatype.
    ValueError
        If `n` is less than zero or greater than the sum of `counts`.

    Raises
    ------
    EfficiencyWarning
        If the accelerated code isn't present or hasn't been compiled.

    Notes
    -----
    If subsampling is performed without replacement (``replace=False``), a copy
    of `counts` is returned if `n` is equal to the number of items in `counts`,
    as all items will be chosen from the original vector.

    If subsampling is performed with replacement (``replace=True``) and `n` is
    equal to the number of items in `counts`, the subsampled vector that is
    returned may not necessarily be the same vector as `counts`.

    Examples
    --------
    Subsample 4 items (without replacement) from a vector of counts:

    >>> import numpy as np
    >>> from skbio.math.subsample import subsample
    >>> a = np.array([4, 5, 0, 2, 1])
    >>> sub = subsample(a, 4)
    >>> sub.sum()
    4
    >>> sub.shape
    (5,)

    Trying to subsample an equal number of items (without replacement) results
    in the same vector as our input:

    >>> subsample([0, 3, 0, 1], 4)
    array([0, 3, 0, 1])

    Subsample 5 items (with replacement):

    >>> sub = subsample([1, 0, 1, 2, 2, 3, 0, 1], 5, replace=True)
    >>> sub.sum()
    5
    >>> sub.shape
    (8,)

    """
    if n < 0:
        raise ValueError("n cannot be negative.")

    counts = np.asarray(counts)
    counts = counts.astype(int, casting='safe')

    if counts.ndim != 1:
        raise ValueError("Only 1-D vectors are supported.")

    counts_sum = counts.sum()
    if n > counts_sum:
        raise ValueError("Cannot subsample more items than exist in input "
                         "counts vector.")

    if replace:
        probs = counts / counts_sum
        result = np.random.multinomial(n, probs)
    else:
        if counts_sum == n:
            result = counts
        else:
            try:
                result = _subsample_without_replacement(counts, n, counts_sum)
            except NameError:
                warn("Accelerated subsampling without replacement isn't"
                     " available.", EfficiencyWarning)

                nz = counts.nonzero()[0]
                unpacked = np.concatenate([np.repeat(np.array(i,), counts[i])
                                           for i in nz])
                permuted = np.random.permutation(unpacked)[:n]

                result = np.zeros(len(counts), dtype=int)
                for p in permuted:
                    result[p] += 1

    return result
<|endoftext|>