<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

from __future__ import unicode_literals
from __future__ import division
from __future__ import absolute_import
from __future__ import print_function

import logging

import numpy as np
import matplotlib.pylab as plt
from mpl_toolkits.mplot3d import Axes3D
import pandas as pd

from sklearn.gaussian_process import GaussianProcess
from sklearn.decomposition import PCA
import warnings

from .lap_cost_matrix import LAPSolver, CMSSolver
from .lapjv import lapjv
from .utils.progress import pprogress

log = logging.getLogger(__name__)

DEFAULTS = {'max_disp':0.1,
            'window_gap':10,
            'sigma':1.,
            'ndims':3,
            'gp_corr':'squared_exponential',
            'gp_regr':'quadratic',
            'gp_theta0':0.1}

class LAPTracker(object):

    def __init__(self, track_df=None,
                 hdfstore=None,
                 dist_function=np.square,
                 params=DEFAULTS,
                 verbose=True):

        if not verbose:
            log.disabled = True
        else:
            log.disabled = False

        self.track = track_df
        self.store = hdfstore
        self.load_parameters(params)
        self.dist_function = dist_function
        self.pos_solver = LAPSolver(self, verbose=verbose)

    def load_parameters(self, params):
        """
        """
        self.params = params

        # Complete the parameter by the defaults
        for key, value in DEFAULTS.items():
            if key not in self.params.keys():
                self.params[key] = value
        self.gp_kwargs = {}
        for key, value in self.params.items():
            if isinstance(key, str) or isinstance(key, unicode) :
                if key.startswith('gp_'):
                    self.gp_kwargs[key[3:]] = value
                else:
                    self.__setattr__(key, value)
    @property
    def times(self):
        '''Unique values of the level 0 index of `self.track`'''
        return self.track.index.get_level_values(0).unique()

    @property
    def labels(self):
        '''Unique values of the level 1 index of `self.track`'''
        return self.track.index.get_level_values(1).unique()

    def get_track(self, verbose=False, save=True, **kwargs):

        for key, value in kwargs.items():
            if key.startswith('gp_'):
                self.gp_kwargs[key[3:]] = value
            else:
                self.__setattr__(key, value)

        log.info('Get track (predict=%s)' % str(self.predict))

        self.track['new_label'] = self.track.index.get_level_values(1)
        time_points = self.times

        n = len(time_points) - 1
        for i, (t0, t1) in enumerate(zip(time_points[:-1], time_points[1:])):
            if verbose:
                pprogress(i / n * 100)
            self.position_track(t0, t1)

        if verbose:
            pprogress(-1)

        self.track.set_index('new_label', append=True, inplace=True)
        self.track.reset_index(level='label', drop=True, inplace=True)
        self.track.index.names[1] = 'label'

        if save:
            self.save_df(self.track, 'sorted')

    def save_df(self, dataframe, name):
        try:
            self.store.open()
            self.store[name] = dataframe
            self.store.close()
        except AttributeError:
            warnings.warn('''No store has been provided, can't save''')

    def reverse_track(self):

        self.track['rev_times'] = self.track.index.get_level_values(0)
        self.track['rev_times'] = (self.track['rev_times'].iloc[-1]
                                   - self.track['rev_times'])
        self.track = self.track.iloc[::-1]
        self.track.set_index('rev_times', append=True, inplace=True,
                             drop='True')
        self.track.reset_index(level='t', drop=True, inplace=True)
        self.track = self.track.swaplevel(0, 1, axis=0)
        self.track.index.names[0] = 't'

    def close_merge_split(self, verbose=False,
                          gap_close_only=True, save=True):

        self.cms_solver = CMSSolver(self, verbose=verbose)

        in_links, out_links = self.cms_solver.solve(gap_close_only=gap_close_only)
        n_segments = len(self.cms_solver.segments)
        n_seeds = len(self.cms_solver.seeds)
        sm_start = n_segments
        sm_stop = n_segments + n_seeds if not gap_close_only else n_segments

        ## Now for gap closing
        old_labels = self.track.index.get_level_values(1).values
        new_labels = old_labels.copy()
        unique_old = np.unique(old_labels)
        unique_new = np.unique(new_labels)
        for n, idx_in in enumerate(out_links[:n_segments]):
            ## gap closing
            if idx_in < n_segments:
                new_label  = unique_new[idx_in]
                unique_new[n] = new_label
            elif idx_in >= sm_stop:
                unique_new[n] = unique_new.max() + 1
        for old, new in zip(unique_old, unique_new):
            new_labels[old_labels == old] = new

        self.track['new_label'] = new_labels
        self.track.set_index('new_label', append=True, inplace=True)
        self.track.reset_index(level='label', drop=True, inplace=True)
        self.track.index.names[1] = 'label'

        ## First split and merge, because this changes
        ## data length, without changing the unique labels

        labels = self.labels
        for n, idx_in in enumerate(out_links[:n_segments]):
            ## splitting
            if n_segments <= idx_in < sm_stop:
                seed = self.cms_solver.seeds[idx_in - sm_start]
                print('split seed %s' % str(seed))
                root_label = labels[seed[0]]
                split_time = seed[1]
                branch_label = labels[n]
                self.split(root_label, split_time, branch_label)

        for n, idx_in in enumerate(out_links[sm_start:sm_stop]):
            ## merging
            if idx_in < n_segments:
                seed = self.cms_solver.seeds[n]
                print('merge seed %s' % str(seed))
                root_label = labels[seed[0]]
                merge_time = seed[1]
                branch_label = labels[idx_in]
                self.merge(root_label, merge_time, branch_label)
        if save:
            self.save_df(self.track, 'sorted')

    def split(self, root_label, split_time, branch_label):

        root_segment = self.get_segment(root_label)
        duplicated = root_segment.loc[:split_time].copy()
        dup_index = pd.MultiIndex.from_tuples([(t, branch_label)
                                               for t in duplicated.index])
        duplicated.set_index(dup_index, inplace=True)
        self.track = self.track.append(duplicated)
        self.track.sortlevel(0, inplace=True)

    def merge(self, root_label, merge_time, branch_label):

        root_segment = self.get_segment(root_label)
        duplicated = root_segment.loc[merge_time:].copy()
        dup_index = pd.MultiIndex.from_tuples([(t, branch_label)
                                               for t in duplicated.index])
        duplicated.set_index(dup_index, inplace=True)
        self.track = self.track.append(duplicated)
        self.track.sortlevel(0, inplace=True)

    def position_track(self, t0, t1):

        coordinates = ['x', 'y'] if self.ndims == 2 else ['x', 'y', 'z']
        pos1 = self.track.loc[t1][coordinates]
        if self.predict:
            pos0, mse0 = self.predict_positions(t0, t1)
        else:
            pos0 = self.track.loc[t0][coordinates]

        try:
            in_links, out_links = self.pos_solver.solve(pos0, pos1)
        except AssertionError:
            warnings.warn('''Someting's' amiss between points %s and %s'''
                          % (t0, t1), RuntimeWarning)
            for n in range(pos1.shape[0]):
                new_label = self.track['new_label'].max() + 1
                self.track.xs(t1)['new_label'].iloc[n] = new_label
            return

        for n, idx_in in enumerate(out_links[:pos1.shape[0]]):
            if idx_in >= pos0.shape[0]:
                # new segment
                new_label = self.track['new_label'].max() + 1
            else:
                new_label  = self.track.loc[t0]['new_label'].iloc[idx_in]
            self.track.loc[t1]['new_label'].iloc[n] = new_label

    def predict_positions(self, t0, t1):
        """
        """

        coordinates = ['x', 'y'] if self.ndims == 2 else ['x', 'y', 'z']
        pos0 = self.track.xs(t0)[coordinates]
        mse0 = pos0.copy() * 0.

        if np.where(self.times == t1) < 3:
            return pos0, mse0
        for lbl in self.labels:
            try:
                segment = self.get_segment(lbl).loc[:t0]
            except KeyError:
                continue
            if segment.shape[0] == 0:
                continue
            if not t0 in segment.index:
                continue
            times = segment.index.get_level_values(0)
            if times.size < 3:
                pos = segment[coordinates].loc[t0]
                mse = pos * 0
            else:
                pred = [_predict_coordinate(segment, coord, times,
                                            t1, self.sigma,
                                            **self.gp_kwargs)
                        for coord in coordinates]
                pos = [p[0] for p in pred]
                mse = [p[1] for p in pred]
            pos0.ix[lbl] = pos
            mse0.ix[lbl] = mse
        return pos0, mse0

    def remove_shorts(self, min_length=3):
        labels = self.track.index.get_level_values(1).unique()
        for lbl in labels:
            segment = self.get_segment(lbl)
            if segment.shape[0] < min_length:
                self.track = self.track.drop([lbl,], level=1)

    def get_segment(self, lbl):
        return self.track.xs(lbl, level=1)

    def segments(self):
        for lbl in self.labels:
            yield self.get_segment(lbl)

    def show(self, ndims=2, **kwargs):

        if ndims == 3:
            fig, axes = plt.subplots(1, 2, subplot_kw={'projection':'3d'})
        else:
            fig, axes = plt.subplots(1, 2)
        ax0, ax1 = axes
        for label in self.labels:
            if ndims == 3:
                ax0, ax1 = self.show_segment_3D(label, axes, **kwargs)
            else:
                ax0, ax1 = self.show_segment_2D(label, axes, **kwargs)
        return ax0, ax1

    def show_3D(self, **kwargs):
        return self.show(ndims=3, **kwargs)

    def show_segment_3D(self, label, axes=None, coords=('x', 'y', 'z')):

        if axes is None:
            fig, axes = plt.subplots(1, 2, subplot_kw={'projection':'3d'})
        ax0, ax1 = axes
        segment = self.get_segment(label)

        xs = segment[coords[0]].values
        ys = segment[coords[1]].values
        zs = segment[coords[2]].values

        times = segment.index.get_level_values(0)
        ax0.plot(times, xs, ys)
        colors = plt.cm.jet(xs.size)
        ax1.plot(xs, ys, zs)
        ax1.scatter(xs , ys, zs, c=colors)
        ax0.set_xlabel('Time (min)')
        ax0.set_ylabel(u'x position (µm)')
        ax0.set_zlabel(u'y position (µm)')
        ax1.set_xlabel(u'x position (µm)')
        ax1.set_ylabel(u'y position (µm)')
        ax1.set_zlabel(u'z position (µm)')

        return ax0, ax1

    def show_segment_2D(self, label, axes=None, coords=('x', 'y')):
        if axes is None:
            fig, axes = plt.subplots(1, 2)
        ax0, ax1 = axes
        segment = self.get_segment(label)
        xs = segment[coords[0]].values
        ys = segment[coords[1]].values

        times = segment.index
        ax0.plot(times, xs)
        colors = plt.cm.jet(xs.size)
        ax1.plot(xs, ys)
        ax1.scatter(xs, ys,
                    c=colors)
        ax0.set_xlabel('Time (min)')
        ax0.set_ylabel(u'x position (µm)')
        ax1.set_xlabel(u'x position (µm)')
        ax1.set_ylabel(u'y position (µm)')
        return ax0, ax1

    def do_pca(self, df=None, ndims=3):

        if not df:
            df = self.track

        self.pca = PCA()
        if ndims == 2:
            coords = ['x', 'y']
            pca_coords = ['x_pca', 'y_pca']
        elif ndims == 3:
            coords = ['x', 'y', 'z']
            pca_coords = ['x_pca', 'y_pca', 'z_pca']
        rotated = self.pca.fit_transform(df[coords])
        for n, coord in enumerate(pca_coords):
            df[coord] = rotated[:, n]
        return df

def _predict_coordinate(segment, coord, times, t1, sigma=10., **kwargs):

    times = np.atleast_2d(times).T
    prev = segment[coord]
    nugget = (sigma / (prev + sigma)) ** 2
    gp = GaussianProcess(nugget=nugget, **kwargs)
    gp.fit(times, prev)
    return gp.predict(t1, eval_MSE=True)
<commit_msg>	modified:   lap_tracker/lap_tracking.py<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

from __future__ import unicode_literals
from __future__ import division
from __future__ import absolute_import
from __future__ import print_function

import logging

import numpy as np
import matplotlib.pylab as plt
from mpl_toolkits.mplot3d import Axes3D
import pandas as pd

from sklearn.gaussian_process import GaussianProcess
from sklearn.decomposition import PCA
import warnings

from .lap_cost_matrix import LAPSolver, CMSSolver
from .lapjv import lapjv
from .utils.progress import pprogress

log = logging.getLogger(__name__)

DEFAULTS = {'max_disp':0.1,
            'window_gap':10,
            'sigma':1.,
            'ndims':3,
            'gp_corr':'squared_exponential',
            'gp_regr':'quadratic',
            'gp_theta0':0.1}

class LAPTracker(object):

    def __init__(self, track_df=None,
                 hdfstore=None,
                 dist_function=np.square,
                 params=DEFAULTS,
                 verbose=True):

        if not verbose:
            log.disabled = True
        else:
            log.disabled = False

        self.track = track_df
        self.store = hdfstore
        self.load_parameters(params)
        self.dist_function = dist_function
        self.pos_solver = LAPSolver(self, verbose=verbose)

    def load_parameters(self, params):
        """
        """
        self.params = params

        # Complete the parameter by the defaults
        for key, value in DEFAULTS.items():
            if key not in self.params.keys():
                self.params[key] = value
        self.gp_kwargs = {}
        for key, value in self.params.items():
            if isinstance(key, str) or isinstance(key, unicode) :
                if key.startswith('gp_'):
                    self.gp_kwargs[key[3:]] = value
                else:
                    self.__setattr__(key, value)
    @property
    def times(self):
        '''Unique values of the level 0 index of `self.track`'''
        return self.track.index.get_level_values(0).unique()

    @property
    def labels(self):
        '''Unique values of the level 1 index of `self.track`'''
        return self.track.index.get_level_values(1).unique()

    def get_track(self, verbose=False, save=True, **kwargs):

        for key, value in kwargs.items():
            if key.startswith('gp_'):
                self.gp_kwargs[key[3:]] = value
            else:
                self.__setattr__(key, value)

        log.info('Get track (predict=%s)' % str(self.predict))

        self.track['new_label'] = self.track.index.get_level_values(1)
        time_points = self.times

        n = len(time_points) - 1
        for i, (t0, t1) in enumerate(zip(time_points[:-1], time_points[1:])):
            if verbose:
                pprogress(i / n * 100)
            self.position_track(t0, t1)

        if verbose:
            pprogress(-1)

        self.track.set_index('new_label', append=True, inplace=True)
        self.track.reset_index(level='label', drop=True, inplace=True)
        self.track.index.names[1] = 'label'

        if save:
            self.save_df(self.track, 'sorted')

    def save_df(self, dataframe, name):
        try:
            self.store.open()
            self.store[name] = dataframe
            self.store.close()
        except AttributeError:
            warnings.warn('''No store has been provided, can't save''')

    def reverse_track(self):

        self.track['rev_times'] = self.track.index.get_level_values(0)
        self.track['rev_times'] = (self.track['rev_times'].iloc[-1]
                                   - self.track['rev_times'])
        self.track = self.track.iloc[::-1]
        self.track.set_index('rev_times', append=True, inplace=True,
                             drop='True')
        self.track.reset_index(level='t', drop=True, inplace=True)
        self.track = self.track.swaplevel(0, 1, axis=0)
        self.track.index.names[0] = 't'

    def close_merge_split(self, verbose=False,
                          gap_close_only=True, save=True):

        self.cms_solver = CMSSolver(self, verbose=verbose)

        in_links, out_links = self.cms_solver.solve(gap_close_only=gap_close_only)
        n_segments = len(self.cms_solver.segments)
        n_seeds = len(self.cms_solver.seeds)
        sm_start = n_segments
        sm_stop = n_segments + n_seeds if not gap_close_only else n_segments

        ## Now for gap closing
        old_labels = self.track.index.get_level_values(1).values
        new_labels = old_labels.copy()
        unique_old = np.unique(old_labels)
        unique_new = np.unique(new_labels)
        for n, idx_in in enumerate(out_links[:n_segments]):
            ## gap closing
            if idx_in < n_segments:
                new_label  = unique_new[idx_in]
                unique_new[n] = new_label
            elif idx_in >= sm_stop:
                unique_new[n] = unique_new.max() + 1
        for old, new in zip(unique_old, unique_new):
            new_labels[old_labels == old] = new

        self.track['new_label'] = new_labels
        self.track.set_index('new_label', append=True, inplace=True)
        self.track.reset_index(level='label', drop=True, inplace=True)
        self.track.index.names[1] = 'label'

        ## First split and merge, because this changes
        ## data length, without changing the unique labels

        labels = self.labels
        for n, idx_in in enumerate(out_links[:n_segments]):
            ## splitting
            if n_segments <= idx_in < sm_stop:
                seed = self.cms_solver.seeds[idx_in - sm_start]
                print('split seed %s' % str(seed))
                root_label = labels[seed[0]]
                split_time = seed[1]
                branch_label = labels[n]
                self.split(root_label, split_time, branch_label)

        for n, idx_in in enumerate(out_links[sm_start:sm_stop]):
            ## merging
            if idx_in < n_segments:
                seed = self.cms_solver.seeds[n]
                print('merge seed %s' % str(seed))
                root_label = labels[seed[0]]
                merge_time = seed[1]
                branch_label = labels[idx_in]
                self.merge(root_label, merge_time, branch_label)
        if save:
            self.save_df(self.track, 'sorted')

    def split(self, root_label, split_time, branch_label):

        root_segment = self.get_segment(root_label)
        duplicated = root_segment.loc[:split_time].copy()
        dup_index = pd.MultiIndex.from_tuples([(t, branch_label)
                                               for t in duplicated.index])
        duplicated.set_index(dup_index, inplace=True)
        self.track = self.track.append(duplicated)
        self.track.sortlevel(0, inplace=True)

    def merge(self, root_label, merge_time, branch_label):

        root_segment = self.get_segment(root_label)
        duplicated = root_segment.loc[merge_time:].copy()
        dup_index = pd.MultiIndex.from_tuples([(t, branch_label)
                                               for t in duplicated.index])
        duplicated.set_index(dup_index, inplace=True)
        self.track = self.track.append(duplicated)
        self.track.sortlevel(0, inplace=True)

    def position_track(self, t0, t1):

        coordinates = ['x', 'y'] if self.ndims == 2 else ['x', 'y', 'z']
        pos1 = self.track.loc[t1][coordinates]
        if self.predict:
            pos0, mse0 = self.predict_positions(t0, t1)
        else:
            pos0 = self.track.loc[t0][coordinates]

        try:
            in_links, out_links = self.pos_solver.solve(pos0, pos1)
        except AssertionError:
            warnings.warn('''Someting's' amiss between points %s and %s'''
                          % (t0, t1), RuntimeWarning)
            for n in range(pos1.shape[0]):
                new_label = self.track['new_label'].max() + 1
                self.track.xs(t1)['new_label'].iloc[n] = new_label
            return

        for n, idx_in in enumerate(out_links[:pos1.shape[0]]):
            if idx_in >= pos0.shape[0]:
                # new segment
                new_label = self.track['new_label'].max() + 1
            else:
                new_label  = self.track.loc[t0]['new_label'].iloc[idx_in]
            self.track.loc[t1]['new_label'].iloc[n] = new_label

    def predict_positions(self, t0, t1):
        """
        """

        coordinates = ['x', 'y'] if self.ndims == 2 else ['x', 'y', 'z']
        pos0 = self.track.xs(t0)[coordinates]
        mse0 = pos0.copy() * 0.

        if np.where(self.times == t1) < 3:
            return pos0, mse0
        for lbl in self.labels:
            try:
                segment = self.get_segment(lbl).loc[:t0]
            except KeyError:
                continue
            if segment.shape[0] == 0:
                continue
            if not t0 in segment.index:
                continue
            times = segment.index.get_level_values(0)
            if times.size < 3:
                pos = segment[coordinates].loc[t0]
                mse = pos * 0
            else:
                pred = [_predict_coordinate(segment, coord, times,
                                            t1, self.sigma,
                                            **self.gp_kwargs)
                        for coord in coordinates]
                pos = [p[0] for p in pred]
                mse = [p[1] for p in pred]
            pos0.ix[lbl] = pos
            mse0.ix[lbl] = mse
        return pos0, mse0

    def remove_shorts(self, min_length=3):
        labels = self.track.index.get_level_values(1).unique()
        for lbl in labels:
            segment = self.get_segment(lbl)
            if segment.shape[0] < min_length:
                self.track = self.track.drop([lbl,], level=1)

    def get_segment(self, lbl):
        return self.track.xs(lbl, level=1)

    def segments(self):
        for lbl in self.labels:
            yield self.get_segment(lbl)

    def show(self, ndims=2, **kwargs):

        if ndims == 3:
            fig, axes = plt.subplots(1, 2, subplot_kw={'projection':'3d'})
        else:
            fig, axes = plt.subplots(1, 2)
        ax0, ax1 = axes
        for label in self.labels:
            if ndims == 3:
                ax0, ax1 = self.show_segment_3D(label, axes, **kwargs)
            else:
                ax0, ax1 = self.show_segment_2D(label, axes, **kwargs)
        return ax0, ax1

    def show_3D(self, **kwargs):
        return self.show(ndims=3, **kwargs)

    def show_segment_3D(self, label, axes=None, coords=('x', 'y', 'z')):

        if axes is None:
            fig, axes = plt.subplots(1, 2, subplot_kw={'projection':'3d'})
        ax0, ax1 = axes
        segment = self.get_segment(label)

        xs = segment[coords[0]].values
        ys = segment[coords[1]].values
        zs = segment[coords[2]].values

        times = segment.index.get_level_values(0)
        ax0.plot(times, xs, ys)
        colors = plt.cm.jet(xs.size)
        ax1.plot(xs, ys, zs)
        ax1.scatter(xs , ys, zs, c=colors)
        ax0.set_xlabel('Time (min)')
        ax0.set_ylabel(u'x position (µm)')
        ax0.set_zlabel(u'y position (µm)')
        ax1.set_xlabel(u'x position (µm)')
        ax1.set_ylabel(u'y position (µm)')
        ax1.set_zlabel(u'z position (µm)')

        return ax0, ax1

    def show_segment_2D(self, label, axes=None, coords=('x', 'y')):
        if axes is None:
            fig, axes = plt.subplots(1, 2)
        ax0, ax1 = axes
        segment = self.get_segment(label)
        xs = segment[coords[0]].values
        ys = segment[coords[1]].values

        times = segment.index
        ax0.plot(times, xs)
        colors = plt.cm.jet(xs.size)
        ax1.plot(xs, ys)
        ax1.scatter(xs, ys,
                    c=colors)
        ax0.set_xlabel('Time (min)')
        ax0.set_ylabel(u'x position (µm)')
        ax1.set_xlabel(u'x position (µm)')
        ax1.set_ylabel(u'y position (µm)')
        return ax0, ax1

    def do_pca(self, df=None, ndims=3, coords=['x', 'y', 'z']):

        if not df:
            df = self.track

        self.pca = PCA()
        pca_coords = [c+'_pca' for c in coords]
        if ndims == 2:
            coords = coords[:2]
            pca_coords = pca_coords[:2]

        rotated = self.pca.fit_transform(df[coords])
        for n, coord in enumerate(pca_coords):
            df[coord] = rotated[:, n]
        return df

def _predict_coordinate(segment, coord, times, t1, sigma=10., **kwargs):

    times = np.atleast_2d(times).T
    prev = segment[coord]
    nugget = (sigma / (prev + sigma)) ** 2
    gp = GaussianProcess(nugget=nugget, **kwargs)
    gp.fit(times, prev)
    return gp.predict(t1, eval_MSE=True)
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

import os
import re
import requests
import json
import time

from settings import DeviceToken
from datetime import datetime

W1_DEVICES = '/sys/bus/w1/devices/'
W1_SENSOR_PATTERN = re.compile('(10|22|28)-.+', re.IGNORECASE)

def sensor_full_path(sensor):
    return os.path.join(W1_DEVICES, sensor)

def find_sensors():
    return [x for x in os.listdir(W1_DEVICES)
            if W1_SENSOR_PATTERN.match(x) and os.path.isdir(sensor_full_path(x))]

def read_sensor(address):
    readings = read_whole_file(os.path.join(W1_DEVICES, address, 'w1_slave'))
    temp_token = 't='
    temp_index = readings.find(temp_token)
    if temp_index < 0:
        return 0.0
    temp = readings[temp_index + len(temp_token):]
    return address, float(temp) / 1000

def read_whole_file(path):
    with open(path, 'r') as f:
        return f.read()

def read_sensors():
    return [read_sensor(x) for x in map(lambda s: s, find_sensors())]

def get_device(token):
    res = requests.get('http://stage.cloud4rpi.io:3000/api/device/{0}/'.format(token))
    ensure_authenticated(res)
    return ServerDevice(res.json())

def put_device(token, device):
    res = requests.put('http://stage.cloud4rpi.io:3000/api/device/{0}/'.format(token),
                       headers={'api_key': token},
                       data=device.dump())
    ensure_authenticated(res)
    return ServerDevice(res.json())

def post_stream(token, stream):
    print 'sending {0} at {1}'.format(stream['payload'], datetime.fromtimestamp(stream['ts']).isoformat())

    res = requests.post('http://stage.cloud4rpi.io:3000/api/device/{0}/stream/'.format(token),
                        headers={'api_key': token},
                        data=json.dumps(stream))
    ensure_authenticated(res)
    return res.json()

def ensure_authenticated(res):
    if res.status_code == 401:
        raise AuthenticationError

class AuthenticationError(Exception):
    pass

class ServerDevice:
    def __init__(self, device_json):
        self.json = device_json
        self.addresses = None
        self.__extract_addresses()

    def sensor_addrs(self):
        return self.addresses

    def add_sensors(self, sensors):
        self.json['sensors'] += map(lambda x: {'name': x, 'address': x}, sensors)
        self.__extract_addresses()

    def whats_new(self, sensors):
        existing = self.sensor_addrs()
        return set(sensors) - set(existing)

    def dump(self):
        return json.dumps(self.json)

    def map_sensors(self, readings):
        # FIXME: Add checks on key presence in index
        index = {sensor['address']: sensor['_id'] for sensor in self.json.get('sensors')}
        return [{index[address]: reading} for address, reading in readings]

    def __extract_addresses(self):
        self.addresses = [sensor['address'] for sensor in self.json['sensors']]

class RpiDaemon:
    def __init__(self):
        self.sensors = None
        self.me = None
        self.token = DeviceToken

    def run(self):
        self.prepare_sensors()
        self.poll()

    def prepare_sensors(self):
        self.know_thyself()
        self.find_sensors()
        self.register_new_sensors()  # if any

    def know_thyself(self):
        self.me = get_device(self.token)

    def find_sensors(self):
        self.sensors = find_sensors()

    def register_new_sensors(self):
        new_sensors = self.me.whats_new(self.sensors)
        if len(new_sensors) > 0:
            self.me.add_sensors(new_sensors)
            self.me = put_device(self.token, self.me)

    def poll(self):
        # TODO:
        # run an infinite loop with sleep
        # catch exceptions, break the on unauth exceptions
        pass

    def tick(self):
        ts = int(time.time())
        readings = read_sensors()
        payload = self.me.map_sensors(readings)
        stream = {
            'ts': ts,
            'payload': payload
        }
        post_stream(self.token, stream)

def modprobe(module):
    ret = os.system('modprobe {0}'.format(module))
    if ret != 0:
        raise EnvironmentError

if __name__ == "__main__":
    try:
        modprobe('w1-gpio')
        modprobe('w1-therm')
    except EnvironmentError:
        print 'Try "sudo python cloud4rpi.py"'
        exit(1)
    except Exception as e:
        print 'Unexpected error: {0}'.format(e.message)
        print 'Terminating...'
        exit(1)

    print 'Starting...'
    daemon = RpiDaemon()
    try:
        daemon.run()
    except AuthenticationError:
        print 'Authentication failed. Check your device token.'
        print 'Terminating...'
        exit(1)
    except Exception as e:
        print 'Unexpected error: {0}'.format(e.message)
        print 'Terminating...'
        exit(1)
<commit_msg>Add polling<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

import os
import re
import requests
import json
import time

from datetime import datetime
from settings import DeviceToken

import settings_vendor as config

W1_DEVICES = '/sys/bus/w1/devices/'
W1_SENSOR_PATTERN = re.compile('(10|22|28)-.+', re.IGNORECASE)

def sensor_full_path(sensor):
    return os.path.join(W1_DEVICES, sensor)

def find_sensors():
    return [x for x in os.listdir(W1_DEVICES)
            if W1_SENSOR_PATTERN.match(x) and os.path.isdir(sensor_full_path(x))]

def read_sensor(address):
    readings = read_whole_file(os.path.join(W1_DEVICES, address, 'w1_slave'))
    temp_token = 't='
    temp_index = readings.find(temp_token)
    if temp_index < 0:
        return 0.0
    temp = readings[temp_index + len(temp_token):]
    return address, float(temp) / 1000

def read_whole_file(path):
    with open(path, 'r') as f:
        return f.read()

def read_sensors():
    return [read_sensor(x) for x in map(lambda s: s, find_sensors())]

def get_device(token):
    res = requests.get('http://stage.cloud4rpi.io:3000/api/device/{0}/'.format(token))
    ensure_authenticated(res)
    return ServerDevice(res.json())

def put_device(token, device):
    res = requests.put('http://stage.cloud4rpi.io:3000/api/device/{0}/'.format(token),
                       headers={'api_key': token},
                       data=device.dump())
    ensure_authenticated(res)
    return ServerDevice(res.json())

def post_stream(token, stream):
    print 'sending {0} at {1}'.format(stream['payload'], datetime.fromtimestamp(stream['ts']).isoformat())

    res = requests.post('http://stage.cloud4rpi.io:3000/api/device/{0}/stream/'.format(token),
                        headers={'api_key': token},
                        data=json.dumps(stream))
    ensure_authenticated(res)
    return res.json()

def ensure_authenticated(res):
    if res.status_code == 401:
        raise AuthenticationError

class AuthenticationError(Exception):
    pass

class ServerDevice:
    def __init__(self, device_json):
        self.json = device_json
        self.addresses = None
        self.__extract_addresses()

    def sensor_addrs(self):
        return self.addresses

    def add_sensors(self, sensors):
        self.json['sensors'] += map(lambda x: {'name': x, 'address': x}, sensors)
        self.__extract_addresses()

    def whats_new(self, sensors):
        existing = self.sensor_addrs()
        return set(sensors) - set(existing)

    def dump(self):
        return json.dumps(self.json)

    def map_sensors(self, readings):
        # FIXME: Add checks on key presence in index
        index = {sensor['address']: sensor['_id'] for sensor in self.json.get('sensors')}
        return [{index[address]: reading} for address, reading in readings]

    def __extract_addresses(self):
        self.addresses = [sensor['address'] for sensor in self.json['sensors']]

class RpiDaemon:
    def __init__(self):
        self.sensors = None
        self.me = None
        self.token = DeviceToken

    def run(self):
        self.prepare_sensors()
        self.poll()

    def prepare_sensors(self):
        self.know_thyself()
        self.find_sensors()
        self.register_new_sensors()  # if any

    def know_thyself(self):
        self.me = get_device(self.token)

    def find_sensors(self):
        self.sensors = find_sensors()

    def register_new_sensors(self):
        new_sensors = self.me.whats_new(self.sensors)
        if len(new_sensors) > 0:
            self.me.add_sensors(new_sensors)
            self.me = put_device(self.token, self.me)

    def poll(self):
        while True:
            self.tick()
            time.sleep(config.scanInterval)

    def tick(self):
        ts = int(time.time())
        readings = read_sensors()
        payload = self.me.map_sensors(readings)
        stream = {
            'ts': ts,
            'payload': payload
        }
        post_stream(self.token, stream)

def modprobe(module):
    ret = os.system('modprobe {0}'.format(module))
    if ret != 0:
        raise EnvironmentError

if __name__ == "__main__":
    try:
        modprobe('w1-gpio')
        modprobe('w1-therm')
    except EnvironmentError:
        print 'Try "sudo python cloud4rpi.py"'
        exit(1)
    except Exception as e:
        print 'Unexpected error: {0}'.format(e.message)
        print 'Terminating...'
        exit(1)

    print 'Starting...'
    daemon = RpiDaemon()
    try:
        daemon.run()
    except AuthenticationError:
        print 'Authentication failed. Check your device token.'
        print 'Terminating...'
        exit(1)
    except Exception as e:
        print 'Unexpected error: {0}'.format(e.message)
        print 'Terminating...'
        exit(1)
<|endoftext|>


<commit_before>#
# Copyright 2014 David Novakovic
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.

from twisted.trial import unittest
from cyclone.escape import xhtml_escape, xhtml_unescape
from cyclone.escape import json_encode, json_decode
from cyclone.escape import squeeze, url_escape, url_unescape
from cyclone.escape import utf8, to_unicode, to_basestring
from cyclone.escape import recursive_unicode

class EscapeTest(unittest.TestCase):
    def test_xhtml_escape(self):
        xml = "<"
        result = xhtml_escape(xml)
        self.assertEqual(result, "<")

    def test_xhtml_unescape(self):
        escaped = "<"
        result = xhtml_unescape(escaped)
        self.assertEqual(result, "<")

    def test_json_encode(self):
        data = {"a": "b"}
        result = json_encode(data)
        self.assertEqual(result, '{"a": "b"}')

    def test_json_decode(self):
        data = '{"a": "b"}'
        result = json_decode(data)
        self.assertEqual(result, {"a": "b"})

    def test_squeeze(self):
        self.assertEqual(squeeze("a  test"), "a test")

    def test_url_escape(self):
        self.assertEqual(url_escape("a value"), "a+value")

    def test_url_unescape(self):
        self.assertEqual(url_unescape("a+value", encoding=None), "a value")
        self.assertEqual(url_unescape("a+value"), "a value")

    def test_utf8(self):
        self.assertEqual(utf8("rawr"), "rawr")
        self.assertEqual(utf8(u"rawr"), "rawr")

    def test_to_unicode(self):
        self.assertEqual(to_unicode("rawr"), u"rawr")
        self.assertEqual(to_unicode(u"rawr"), u"rawr")

    def test_to_basestring(self):
        """
        Not sure this is 100% testable in python 2.
        """
        self.assertEqual(to_basestring("rawr"), "rawr")
        self.assertEqual(to_basestring(u"rawr"), "rawr")

    def test_recursive_unicode(self):
        self.assertEqual(recursive_unicode("rawr"), u"rawr")
        self.assertEqual(
            recursive_unicode({"rawr": "rawr"}), {"rawr": u"rawr"})
        self.assertEqual(
            recursive_unicode(["rawr", "rawr"]), [u"rawr", u"rawr"])
<commit_msg>More escaping tests.<commit_after>#
# Copyright 2014 David Novakovic
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.

from twisted.trial import unittest
from cyclone.escape import xhtml_escape, xhtml_unescape
from cyclone.escape import json_encode, json_decode
from cyclone.escape import squeeze, url_escape, url_unescape
from cyclone.escape import utf8, to_unicode, to_basestring
from cyclone.escape import recursive_unicode, linkify

class EscapeTest(unittest.TestCase):
    def test_xhtml_escape(self):
        xml = "<"
        result = xhtml_escape(xml)
        self.assertEqual(result, "<")

    def test_xhtml_unescape(self):
        escaped = "<"
        result = xhtml_unescape(escaped)
        self.assertEqual(result, "<")

    def test_json_encode(self):
        data = {"a": "b"}
        result = json_encode(data)
        self.assertEqual(result, '{"a": "b"}')

    def test_json_decode(self):
        data = '{"a": "b"}'
        result = json_decode(data)
        self.assertEqual(result, {"a": "b"})

    def test_squeeze(self):
        self.assertEqual(squeeze("a  test"), "a test")

    def test_url_escape(self):
        self.assertEqual(url_escape("a value"), "a+value")

    def test_url_unescape(self):
        self.assertEqual(url_unescape("a+value", encoding=None), "a value")
        self.assertEqual(url_unescape("a+value"), "a value")

    def test_utf8(self):
        self.assertEqual(utf8("rawr"), "rawr")
        self.assertEqual(utf8(u"rawr"), "rawr")

    def test_to_unicode(self):
        self.assertEqual(to_unicode("rawr"), u"rawr")
        self.assertEqual(to_unicode(u"rawr"), u"rawr")

    def test_to_basestring(self):
        """
        Not sure this is 100% testable in python 2.
        """
        self.assertEqual(to_basestring("rawr"), "rawr")
        self.assertEqual(to_basestring(u"rawr"), "rawr")

    def test_recursive_unicode(self):
        self.assertEqual(recursive_unicode("rawr"), u"rawr")
        self.assertEqual(
            recursive_unicode({"rawr": "rawr"}), {"rawr": u"rawr"})
        self.assertEqual(
            recursive_unicode(["rawr", "rawr"]), [u"rawr", u"rawr"])

    def test_linkify(self):
        """
        Rough tests just to ensure we don't have exceptions.
        """
        urls = [
            "http://testing.com/a/long/url/right/here",
        ]
        for u in urls:
            link = linkify(u, shorten=True)
            self.assertTrue(
                link.startswith("<a "), link
            )
        linkify("spdy://testing.com/a/long/url/right/here", shorten=True)
        linkify(
            "http://testing.com/alongurlrighthere"
            "alongurlrighthere"
            "alongurlrighthere"
            "alongurlrighthere"
            "/a/long/url/right/here",
            shorten=True, require_protocol=True, extra_params=lambda x: "x=y")
<|endoftext|>


<commit_before>import numpy as np
import os

import chainer
from chainercv.utils import read_image

def directory_parsing_label_names(root, numerical_sort=False):
    """Get label names from the directories that are named by them.

    The label names are the names of the directories that locate a
    layer below the root directory.

    The label names can be used together with
    :class:`~chainercv.datasets.DirectoryParsingLabelDataset`.
    The index of a label name corresponds to the label id
    that is used by the dataset to refer the label.

    Args:
        root (str): The root directory.
        numerical_sort (bool): Label names are sorted numerically.
            This means that label :obj:`2` is before label :obj:`10`,
            which is not the case when string sort is used.
            The default value is :obj:`False`.

    Returns:
        list of strings:
        Sorted names of classes.

    """
    label_names = [d for d in os.listdir(root)
                   if os.path.isdir(os.path.join(root, d))]

    if not numerical_sort:
        label_names.sort()
    else:
        label_names = sorted(label_names, key=int)
    return label_names

def _check_img_ext(path):
    img_extensions = ['.jpg', '.jpeg', '.png', '.ppm', '.bmp']
    return any(os.path.splitext(path)[1].lower() == extension for
               extension in img_extensions)

def _parse_label_dataset(root, label_names,
                         check_img_file=_check_img_ext):
    img_paths = []
    labels = []
    for label, label_name in enumerate(label_names):
        label_dir = os.path.join(root, label_name)
        if not os.path.isdir(label_dir):
            continue

        walk_dir = sorted(os.walk(label_dir), key=lambda x: x[0])
        for cur_dir, _, names in walk_dir:
            names = sorted(names)
            for name in names:
                img_path = os.path.join(cur_dir, name)
                if check_img_file(img_path):
                    img_paths.append(img_path)
                    labels.append(label)

    return img_paths, np.array(labels, np.int32)

class DirectoryParsingLabelDataset(chainer.dataset.DatasetMixin):
    """A label dataset whose label names are the names of the subdirectories.

    The label names are the names of the directories that locate a layer below
    the root directory.
    All images locating under the subdirectoies will be categorized to classes
    with subdirectory names.
    An image is parsed only when the function :obj:`check_img_file`
    returns :obj:`True` by taking the path to the image as an argument.
    If :obj:`check_img_file` is :obj:`None`,
    the path with any image extensions will be parsed.

    Example:

        A directory structure should be one like below.

        .. code::

            root
            |-- class_0
            |   |-- img_0.png
            |   |-- img_1.png
            |
            --- class_1
                |-- img_0.png

        >>> from chainercv.datasets import DirectoryParsingLabelDataset
        >>> dataset = DirectoryParsingLabelDataset('root')
        >>> dataset.paths
        ['root/class_0/img_0.png', 'root/class_0/img_1.png',
        'root_class_1/img_0.png']
        >>> dataset.labels
        array([0, 0, 1])

    Args:
        root (str): The root directory.
        check_img_file (callable): A function to determine
            if a file should be included in the dataset.
        color (bool): If :obj:`True`, this dataset read images
            as color images. The default value is :obj:`True`.
        numerical_sort (bool): Label names are sorted numerically.
            This means that label :obj:`2` is before label :obj:`10`,
            which is not the case when string sort is used.
            Regardless of this option, string sort is used for the
            order of files with the same label.
            The default value is :obj:`False`.

    """

    def __init__(self, root, check_img_file=None, color=True,
                 numerical_sort=False):
        self.color = color

        label_names = directory_parsing_label_names(
            root, numerical_sort=numerical_sort)
        if check_img_file is None:
            check_img_file = _check_img_ext

        self.img_paths, self.labels = _parse_label_dataset(
            root, label_names, check_img_file)

    def __len__(self):
        return len(self.img_paths)

    def get_example(self, i):
        img = read_image(self.img_paths[i], color=self.color)
        label = self.labels[i]
        return img, label
<commit_msg>update<commit_after>import numpy as np
import os

import chainer

from chainercv.chainer_experimental.datasets.sliceable import GetterDataset
from chainercv.utils import read_image

def directory_parsing_label_names(root, numerical_sort=False):
    """Get label names from the directories that are named by them.

    The label names are the names of the directories that locate a
    layer below the root directory.

    The label names can be used together with
    :class:`~chainercv.datasets.DirectoryParsingLabelDataset`.
    The index of a label name corresponds to the label id
    that is used by the dataset to refer the label.

    Args:
        root (str): The root directory.
        numerical_sort (bool): Label names are sorted numerically.
            This means that label :obj:`2` is before label :obj:`10`,
            which is not the case when string sort is used.
            The default value is :obj:`False`.

    Returns:
        list of strings:
        Sorted names of classes.

    """
    label_names = [d for d in os.listdir(root)
                   if os.path.isdir(os.path.join(root, d))]

    if not numerical_sort:
        label_names.sort()
    else:
        label_names = sorted(label_names, key=int)
    return label_names

def _check_img_ext(path):
    img_extensions = ['.jpg', '.jpeg', '.png', '.ppm', '.bmp']
    return any(os.path.splitext(path)[1].lower() == extension for
               extension in img_extensions)

def _parse_label_dataset(root, label_names,
                         check_img_file=_check_img_ext):
    img_paths = []
    labels = []
    for label, label_name in enumerate(label_names):
        label_dir = os.path.join(root, label_name)
        if not os.path.isdir(label_dir):
            continue

        walk_dir = sorted(os.walk(label_dir), key=lambda x: x[0])
        for cur_dir, _, names in walk_dir:
            names = sorted(names)
            for name in names:
                img_path = os.path.join(cur_dir, name)
                if check_img_file(img_path):
                    img_paths.append(img_path)
                    labels.append(label)

    return img_paths, np.array(labels, np.int32)

class DirectoryParsingLabelDataset(GetterDataset):
    """A label dataset whose label names are the names of the subdirectories.

    The label names are the names of the directories that locate a layer below
    the root directory.
    All images locating under the subdirectoies will be categorized to classes
    with subdirectory names.
    An image is parsed only when the function :obj:`check_img_file`
    returns :obj:`True` by taking the path to the image as an argument.
    If :obj:`check_img_file` is :obj:`None`,
    the path with any image extensions will be parsed.

    Example:

        A directory structure should be one like below.

        .. code::

            root
            |-- class_0
            |   |-- img_0.png
            |   |-- img_1.png
            |
            --- class_1
                |-- img_0.png

        >>> from chainercv.datasets import DirectoryParsingLabelDataset
        >>> dataset = DirectoryParsingLabelDataset('root')
        >>> dataset.paths
        ['root/class_0/img_0.png', 'root/class_0/img_1.png',
        'root_class_1/img_0.png']
        >>> dataset.labels
        array([0, 0, 1])

    Args:
        root (str): The root directory.
        check_img_file (callable): A function to determine
            if a file should be included in the dataset.
        color (bool): If :obj:`True`, this dataset read images
            as color images. The default value is :obj:`True`.
        numerical_sort (bool): Label names are sorted numerically.
            This means that label :obj:`2` is before label :obj:`10`,
            which is not the case when string sort is used.
            Regardless of this option, string sort is used for the
            order of files with the same label.
            The default value is :obj:`False`.

    This dataset returns the following data.

    .. csv-table::
        :header: name, shape, dtype, format

        :obj:`img`, ":math:`(3, H, W)` [#directory_parsing_1]_", \
        :obj:`float32`, "RGB, :math:`[0, 255]`"
        :obj:`label`, scalar, :obj:`int32`, ":math:`[0, \#class - 1]`"

    .. [#directory_parsing_1] :math:`(1, H, W)` if :obj:`color = False`.
    """

    def __init__(self, root, check_img_file=None, color=True,
                 numerical_sort=False):
        super(DirectoryParsingLabelDataset, self).__init__()

        self.color = color

        label_names = directory_parsing_label_names(
            root, numerical_sort=numerical_sort)
        if check_img_file is None:
            check_img_file = _check_img_ext

        self.img_paths, self.labels = _parse_label_dataset(
            root, label_names, check_img_file)

        self.add_getter('img', lambda i:
                        read_image(self.img_paths[i], color=self.color))
        self.add_getter('label', lambda i: self.labels[i])

    def __len__(self):
        return len(self.img_paths)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

# * Copyright (c) 2009-2018. Authors: see NOTICE file.
# *
# * Licensed under the Apache License, Version 2.0 (the "License");
# * you may not use this file except in compliance with the License.
# * You may obtain a copy of the License at
# *
# *      http://www.apache.org/licenses/LICENSE-2.0
# *
# * Unless required by applicable law or agreed to in writing, software
# * distributed under the License is distributed on an "AS IS" BASIS,
# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# * See the License for the specific language governing permissions and
# * limitations under the License.

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals

import re

__author__ = "Rubens Ulysse <urubens@uliege.be>"
__contributors__ = ["Marée Raphaël <raphael.maree@uliege.be>", "Mormont Romain <r.mormont@uliege.be>"]
__copyright__ = "Copyright 2010-2018 University of Liège, Belgium, http://www.cytomine.be/"

from cytomine.cytomine import Cytomine
from cytomine.models.annotation import Annotation
from cytomine.models.collection import Collection, DomainCollection
from cytomine.models.model import Model, DomainModel

class Property(DomainModel):
    def __init__(self, object, key=None, value=None, **attributes):
        super(Property, self).__init__(object)
        self.key = key
        self.value = value
        self.populate(attributes)
        self._by_key = False

    @property
    def obj(self):
        return self._object

    @obj.setter
    def obj(self, value):
        self._object = value
        if isinstance(value, Annotation):
            self.domainClassName = "annotation"
        else:
            self.domainClassName = value.class_
        self.domainIdent = value.id

    def uri(self):
        if self._by_key and self.domainClassName and self.domainIdent and self.key:
            uri = "domain/{}/{}/key/{}/property.json".format(self.domainClassName, self.domainIdent, self.key)
        else:
            uri = super(Property, self).uri()

        if self.domainClassName == "annotation":
            uri = uri.replace("domain/", "")

        return uri

    def fetch(self, id=None, key=None):
        if self.id is None and id is None and self.key is None and key is None:
            raise ValueError("Cannot fetch a model with no ID and no key.")
        if id is not None:
            self.id = id
        if key is not None:
            self.key = key
            self._by_key = True

        model = Cytomine.get_instance().get_model(self, self.query_parameters)
        self._by_key = False
        return model

    def __str__(self):
        return "[{}] {} : {} ({}) - Key: {} - Value {}".format(self.callback_identifier, self.id, self.domainClassName,
                                                               self.domainIdent, self.key, self.value)

class PropertyCollection(DomainCollection):
    def __init__(self, object, filters=None, max=0, offset=0, **parameters):
        super(PropertyCollection, self).__init__(Property, object, filters, max, offset)
        self._allowed_filters = [None]
        self.set_parameters(parameters)

    def uri(self, without_filters=False):
        uri = super(PropertyCollection, self).uri(without_filters)
        if self._domainClassName == "annotation":
            uri = uri.replace("domain/", "")
        return uri

    def as_dict(self):
        """Transform the property collection into a python dictionary mapping keys
            with their respective Property objects.
        """
        return {p.key: p for p in self}

    @property
    def _obj(self):
        return self._object

    @_obj.setter
    def _obj(self, value):
        self._object = value
        if isinstance(value, Annotation):
            self._domainClassName = "annotation"
        else:
            self._domainClassName = value.class_
        self._domainIdent = value.id

class AttachedFile(DomainModel):
    def __init__(self, object, filename=None, file=None, **attributes):
        super(AttachedFile, self).__init__(object)
        self.filename = filename
        self.file = file
        self.url = None
        self.populate(attributes)

    def uri(self):
        if self.is_new():
            return "{}.json".format(self.callback_identifier)
        else:
            return "{}/{}.json".format(self.callback_identifier, self.id)

    def save(self):
        return self.upload()

    def update(self, id=None, **attributes):
        return self.upload()

    def upload(self):
        return Cytomine.get_instance().upload_file(self, self.file, uri='attachedfile.json',
                                                    query_parameters={"domainClassName": self.domainClassName,
                                                    "domainIdent": self.domainIdent, "filename":self.filename})

    def download(self, destination="{filename}", override=False):
        if self.is_new():
            raise ValueError("Cannot download file if not existing ID.")

        pattern = re.compile("{(.*?)}")
        destination = re.sub(pattern, lambda m: str(getattr(self, str(m.group(0))[1:-1], "_")), destination)

        return Cytomine.get_instance().download_file("{}/{}/download".format(self.callback_identifier, self.id),
                                                     destination, override)

class AttachedFileCollection(DomainCollection):
    def __init__(self, object, filters=None, max=0, offset=0, **parameters):
        super(AttachedFileCollection, self).__init__(AttachedFile, object, filters, max, offset)
        self._allowed_filters = [None]
        self.set_parameters(parameters)

class Description(DomainModel):
    def __init__(self, object, data=None, **attributes):
        super(Description, self).__init__(object)
        self.data = data
        self.populate(attributes)

    def uri(self):
        return "domain/{}/{}/{}.json".format(self._object.class_, self._object.id, self.callback_identifier)

    def fetch(self, id=None):
        if id is not None:
            self.id = id

        return Cytomine.get_instance().get_model(self, self.query_parameters)

class Tag(Model):
    def __init__(self, name=None, **attributes):
        super(Tag, self).__init__()
        self.name = name
        self.populate(attributes)

class TagCollection(Collection):
    def __init__(self, filters=None, max=0, offset=0, **parameters):
        super(TagCollection, self).__init__(Tag, filters, max, offset)
        self._allowed_filters = [None]
        self.set_parameters(parameters)

class TagDomainAssociation(DomainModel):
    def __init__(self, object, tag=None, **attributes):
        super(TagDomainAssociation, self).__init__(object)
        self.tag = tag
        self.populate(attributes)

    def uri(self):
        if self.id:
            uri = "tag_domain_association/{}.json".format(self.id)
        elif self.domainClassName and self.domainIdent:
            uri = super(TagDomainAssociation, self).uri()

        return uri

    @property
    def callback_identifier(self):
        return "tag_domain_association"

class TagDomainAssociationCollection(DomainCollection):
    def __init__(self, object, filters=None, max=0, offset=0, **parameters):
        super(TagDomainAssociationCollection, self).__init__(TagDomainAssociation, object, filters, max, offset)
        self._allowed_filters = [None]
        self.set_parameters(parameters)

    @property
    def callback_identifier(self):
        return "tag_domain_association"
<commit_msg>fix regression introduce by a previous commit<commit_after># -*- coding: utf-8 -*-

# * Copyright (c) 2009-2018. Authors: see NOTICE file.
# *
# * Licensed under the Apache License, Version 2.0 (the "License");
# * you may not use this file except in compliance with the License.
# * You may obtain a copy of the License at
# *
# *      http://www.apache.org/licenses/LICENSE-2.0
# *
# * Unless required by applicable law or agreed to in writing, software
# * distributed under the License is distributed on an "AS IS" BASIS,
# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# * See the License for the specific language governing permissions and
# * limitations under the License.

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals

import re

__author__ = "Rubens Ulysse <urubens@uliege.be>"
__contributors__ = ["Marée Raphaël <raphael.maree@uliege.be>", "Mormont Romain <r.mormont@uliege.be>"]
__copyright__ = "Copyright 2010-2018 University of Liège, Belgium, http://www.cytomine.be/"

from cytomine.cytomine import Cytomine
from cytomine.models.annotation import Annotation
from cytomine.models.collection import Collection, DomainCollection
from cytomine.models.model import Model, DomainModel

class Property(DomainModel):
    def __init__(self, object, key=None, value=None, **attributes):
        super(Property, self).__init__(object)
        self.key = key
        self.value = value
        self.populate(attributes)
        self._by_key = False

    @property
    def obj(self):
        return self._object

    @obj.setter
    def obj(self, value):
        self._object = value
        if isinstance(value, Annotation):
            self.domainClassName = "annotation"
        else:
            self.domainClassName = value.class_
        self.domainIdent = value.id

    def uri(self):
        if self._by_key and self.domainClassName and self.domainIdent and self.key:
            uri = "domain/{}/{}/key/{}/property.json".format(self.domainClassName, self.domainIdent, self.key)
        else:
            uri = super(Property, self).uri()

        if self.domainClassName == "annotation":
            uri = uri.replace("domain/", "")

        return uri

    def fetch(self, id=None, key=None):
        if self.id is None and id is None and self.key is None and key is None:
            raise ValueError("Cannot fetch a model with no ID and no key.")
        if id is not None:
            self.id = id
        if key is not None:
            self.key = key
            self._by_key = True

        model = Cytomine.get_instance().get_model(self, self.query_parameters)
        self._by_key = False
        return model

    def __str__(self):
        return "[{}] {} : {} ({}) - Key: {} - Value {}".format(self.callback_identifier, self.id, self.domainClassName,
                                                               self.domainIdent, self.key, self.value)

class PropertyCollection(DomainCollection):
    def __init__(self, object, filters=None, max=0, offset=0, **parameters):
        super(PropertyCollection, self).__init__(Property, object, filters, max, offset)
        self._allowed_filters = [None]
        self.set_parameters(parameters)

    def uri(self, without_filters=False):
        uri = super(PropertyCollection, self).uri(without_filters)
        if self._domainClassName == "annotation":
            uri = uri.replace("domain/", "")
        return uri

    def as_dict(self):
        """Transform the property collection into a python dictionary mapping keys
            with their respective Property objects.
        """
        return {p.key: p for p in self}

    @property
    def _obj(self):
        return self._object

    @_obj.setter
    def _obj(self, value):
        self._object = value
        if isinstance(value, Annotation):
            self._domainClassName = "annotation"
        else:
            self._domainClassName = value.class_
        self._domainIdent = value.id

class AttachedFile(DomainModel):
    def __init__(self, object, filename=None, file=None, **attributes):
        super(AttachedFile, self).__init__(object)
        self.filename = filename
        self.file = file
        self.url = None
        self.populate(attributes)

    def uri(self):
        if self.is_new():
            return "{}.json".format(self.callback_identifier)
        else:
            return "{}/{}.json".format(self.callback_identifier, self.id)

    def save(self):
        return self.upload()

    def update(self, id=None, **attributes):
        return self.upload()

    def upload(self):
        if self.file:
            return Cytomine.get_instance().upload_file(self, self.file, uri='attachedfile.json',
                                                    query_parameters={"domainClassName": self.domainClassName,
                                                    "domainIdent": self.domainIdent, "filename":self.filename})
        else:
            return Cytomine.get_instance().upload_file(self, self.filename,
                                                    query_parameters={"domainClassName": self.domainClassName,
                                                    "domainIdent": self.domainIdent})

    def download(self, destination="{filename}", override=False):
        if self.is_new():
            raise ValueError("Cannot download file if not existing ID.")

        pattern = re.compile("{(.*?)}")
        destination = re.sub(pattern, lambda m: str(getattr(self, str(m.group(0))[1:-1], "_")), destination)

        return Cytomine.get_instance().download_file("{}/{}/download".format(self.callback_identifier, self.id),
                                                     destination, override)

class AttachedFileCollection(DomainCollection):
    def __init__(self, object, filters=None, max=0, offset=0, **parameters):
        super(AttachedFileCollection, self).__init__(AttachedFile, object, filters, max, offset)
        self._allowed_filters = [None]
        self.set_parameters(parameters)

class Description(DomainModel):
    def __init__(self, object, data=None, **attributes):
        super(Description, self).__init__(object)
        self.data = data
        self.populate(attributes)

    def uri(self):
        return "domain/{}/{}/{}.json".format(self._object.class_, self._object.id, self.callback_identifier)

    def fetch(self, id=None):
        if id is not None:
            self.id = id

        return Cytomine.get_instance().get_model(self, self.query_parameters)

class Tag(Model):
    def __init__(self, name=None, **attributes):
        super(Tag, self).__init__()
        self.name = name
        self.populate(attributes)

class TagCollection(Collection):
    def __init__(self, filters=None, max=0, offset=0, **parameters):
        super(TagCollection, self).__init__(Tag, filters, max, offset)
        self._allowed_filters = [None]
        self.set_parameters(parameters)

class TagDomainAssociation(DomainModel):
    def __init__(self, object, tag=None, **attributes):
        super(TagDomainAssociation, self).__init__(object)
        self.tag = tag
        self.populate(attributes)

    def uri(self):
        if self.id:
            uri = "tag_domain_association/{}.json".format(self.id)
        elif self.domainClassName and self.domainIdent:
            uri = super(TagDomainAssociation, self).uri()

        return uri

    @property
    def callback_identifier(self):
        return "tag_domain_association"

class TagDomainAssociationCollection(DomainCollection):
    def __init__(self, object, filters=None, max=0, offset=0, **parameters):
        super(TagDomainAssociationCollection, self).__init__(TagDomainAssociation, object, filters, max, offset)
        self._allowed_filters = [None]
        self.set_parameters(parameters)

    @property
    def callback_identifier(self):
        return "tag_domain_association"
<|endoftext|>


<commit_before># collectd-cloudstack - cloudstack.py
#
# Author : Antoine Coetsier @ exoscale
# Description : This is a collectd python module to gather stats from cloudstack
# inspired by collectd-haproxy from Michael Leinartas - https://github.com/mleinart/collectd-haproxy

import collectd
import urllib2
import urllib
import json
import hmac
import base64
import hashlib
import re
import CloudStack

NAME = 'cloudstack'

DEFAULT_API = 'http://localh:8096/client/api'
DEFAULT_AUTH = False
DEFAULT_APIKEY = ''
DEFAULT_SECRET = ''
VERBOSE_LOGGING = False

METRIC_TYPES = {
  'memoryused': ('h_memory_used', 'memory'),
  'memorytotal': ('h_memory_total', 'memory'),
  'memoryallocated': ('h_memory_allocated', 'memory'),
  'activeviewersessions': ('console_active_sessions', 'current')
}

METRIC_DELIM = '.'

hypervisors = []

def get_stats():
  stats = dict()

  logger('verb', "get_stats calls API %s KEY %s SECRET %s" % (API_MONITORS, APIKEY_MONITORS, SECRET_MONITORS))
  cloudstack = CloudStack.Client(API_MONITORS, APIKEY_MONITORS, SECRET_MONITORS)
  try:
 	hypervisors = cloudstack.listHosts({
                        'type': 'Routing'
                })
  except:
     	logger('warn', "status err Unable to connect to CloudStack URL at %s" % API_MONITORS)
  for  h in hypervisors:
	metricnameMemUsed = METRIC_DELIM.join([ h['name'].lower(), h['podname'].lower(), h['zonename'].lower(), 'memoryused' ])
	metricnameMemTotal = METRIC_DELIM.join([ h['name'].lower(), h['podname'].lower(), h['zonename'].lower(), 'memorytotal' ])
	metricnameMemAlloc = METRIC_DELIM.join([ h['name'].lower(), h['podname'].lower(), h['zonename'].lower(), 'memoryallocated' ])
	try:
        	stats[metricnameMemUsed] = h['memoryused']
        	stats[metricnameMemTotal] = h['memorytotal']
        	stats[metricnameMemAlloc] = h['memoryallocated']
  		logger('verb', "readings :  %s memory used %s " % (h['name'], h['memoryused']))
	except (TypeError, ValueError), e:
        	pass
  try:
	systemvms = cloudstack.listSystemVms({
		'systemvmtype': 'consoleproxy'
		})
  except:
     	logger('warn', "status err Unable to connect to CloudStack URL at %s" % API_MONITORS)

  for systemvm in systemvms:
	metricnameSessions = METRIC_DELIM.join([ systemvm['name'].lower(), h['podid'].lower(), re.sub(r"\s+", '-', h['zonename'].lower()), 'activeviewersessions' ])
	if 'activeviewersessions' in systemvm:
		stats[metricnameSessions] = systemvm['activeviewersessions']

  return stats

# callback configuration for module
def configure_callback(conf):
  global API_MONITORS, APIKEY_MONITORS, SECRET_MONITORS, AUTH_MONITORS, VERBOSE_LOGGING
  API_MONITORS = ''
  APIKEY_MONITORS = ''
  SECRET_MONITORS = ''
  AUTH_MONITORS = DEFAULT_AUTH
  VERBOSE_LOGGING = False

  for node in conf.children:
    if node.key == "Api":
      API_MONITORS = node.values[0]
    elif node.key == "ApiKey":
      APIKEY_MONITORS = node.values[0]
    elif node.key == "Secret":
      SECRET_MONITORS = node.values[0]
    elif node.key == "Auth":
      AUTH_MONITORS = node.values[0]
    elif node.key == "Verbose":
      VERBOSE_LOGGING = bool(node.values[0])
    else:
      logger('warn', 'Unknown config key: %s' % node.key)

  if not API_MONITORS:
    API_MONITORS += DEFAULT_API

def read_callback():
  logger('verb', "beginning read_callback")
  info = get_stats()

  if not info:
    logger('warn', "%s: No data received" % NAME)
    return

  for key,value in info.items():
    key_prefix = ''
    key_root = key
    logger('verb', "read_callback key %s" % (key))
    logger('verb', "read_callback value %s" % (value))
    if not value in METRIC_TYPES:
      try:
        key_prefix, key_root = key.rsplit(METRIC_DELIM,1)
      except ValueError, e:
        pass
    if not key_root in METRIC_TYPES:
      continue

    key_root, val_type = METRIC_TYPES[key_root]
    key_name = METRIC_DELIM.join([key_prefix, key_root])
    logger('verb', "key_name %s" % (key_name))
    val = collectd.Values(plugin=NAME, type=val_type)
    val.type_instance = key_name
    val.values = [ value ]
    val.dispatch()

# logging function
def logger(t, msg):
    if t == 'err':
        collectd.error('%s: %s' % (NAME, msg))
    elif t == 'warn':
        collectd.warning('%s: %s' % (NAME, msg))
    elif t == 'verb':
        if VERBOSE_LOGGING:
            collectd.info('%s: %s' % (NAME, msg))
    else:
        collectd.notice('%s: %s' % (NAME, msg))
# main
collectd.register_config(configure_callback)
collectd.register_read(read_callback)
<commit_msg>metric formating for zonenames<commit_after># collectd-cloudstack - cloudstack.py
#
# Author : Antoine Coetsier @ exoscale
# Description : This is a collectd python module to gather stats from cloudstack
# inspired by collectd-haproxy from Michael Leinartas - https://github.com/mleinart/collectd-haproxy

import collectd
import urllib2
import urllib
import json
import hmac
import base64
import hashlib
import re
import CloudStack

NAME = 'cloudstack'

DEFAULT_API = 'http://localh:8096/client/api'
DEFAULT_AUTH = False
DEFAULT_APIKEY = ''
DEFAULT_SECRET = ''
VERBOSE_LOGGING = False

METRIC_TYPES = {
  'memoryused': ('h_memory_used', 'memory'),
  'memorytotal': ('h_memory_total', 'memory'),
  'memoryallocated': ('h_memory_allocated', 'memory'),
  'activeviewersessions': ('console_active_sessions', 'current')
}

METRIC_DELIM = '.'

hypervisors = []

def get_stats():
  stats = dict()

  logger('verb', "get_stats calls API %s KEY %s SECRET %s" % (API_MONITORS, APIKEY_MONITORS, SECRET_MONITORS))
  cloudstack = CloudStack.Client(API_MONITORS, APIKEY_MONITORS, SECRET_MONITORS)
  try:
 	hypervisors = cloudstack.listHosts({
                        'type': 'Routing'
                })
  except:
     	logger('warn', "status err Unable to connect to CloudStack URL at %s" % API_MONITORS)
  for  h in hypervisors:
	metricnameMemUsed = METRIC_DELIM.join([ h['name'].lower(), h['podname'].lower(), re.sub(r"\s+", '-', h['zonename'].lower()), 'memoryused' ])
	metricnameMemTotal = METRIC_DELIM.join([ h['name'].lower(), h['podname'].lower(), re.sub(r"\s+", '-', h['zonename'].lower()), 'memorytotal' ])
	metricnameMemAlloc = METRIC_DELIM.join([ h['name'].lower(), h['podname'].lower(), re.sub(r"\s+", '-', h['zonename'].lower()), 'memoryallocated' ])
	try:
        	stats[metricnameMemUsed] = h['memoryused']
        	stats[metricnameMemTotal] = h['memorytotal']
        	stats[metricnameMemAlloc] = h['memoryallocated']
  		logger('verb', "readings :  %s memory used %s " % (h['name'], h['memoryused']))
	except (TypeError, ValueError), e:
        	pass
  try:
	systemvms = cloudstack.listSystemVms({
		'systemvmtype': 'consoleproxy'
		})
  except:
     	logger('warn', "status err Unable to connect to CloudStack URL at %s" % API_MONITORS)

  for systemvm in systemvms:
	metricnameSessions = METRIC_DELIM.join([ systemvm['name'].lower(), h['podid'].lower(), re.sub(r"\s+", '-', h['zonename'].lower()), 'activeviewersessions' ])
	if 'activeviewersessions' in systemvm:
		stats[metricnameSessions] = systemvm['activeviewersessions']

  return stats

# callback configuration for module
def configure_callback(conf):
  global API_MONITORS, APIKEY_MONITORS, SECRET_MONITORS, AUTH_MONITORS, VERBOSE_LOGGING
  API_MONITORS = ''
  APIKEY_MONITORS = ''
  SECRET_MONITORS = ''
  AUTH_MONITORS = DEFAULT_AUTH
  VERBOSE_LOGGING = False

  for node in conf.children:
    if node.key == "Api":
      API_MONITORS = node.values[0]
    elif node.key == "ApiKey":
      APIKEY_MONITORS = node.values[0]
    elif node.key == "Secret":
      SECRET_MONITORS = node.values[0]
    elif node.key == "Auth":
      AUTH_MONITORS = node.values[0]
    elif node.key == "Verbose":
      VERBOSE_LOGGING = bool(node.values[0])
    else:
      logger('warn', 'Unknown config key: %s' % node.key)

  if not API_MONITORS:
    API_MONITORS += DEFAULT_API

def read_callback():
  logger('verb', "beginning read_callback")
  info = get_stats()

  if not info:
    logger('warn', "%s: No data received" % NAME)
    return

  for key,value in info.items():
    key_prefix = ''
    key_root = key
    logger('verb', "read_callback key %s" % (key))
    logger('verb', "read_callback value %s" % (value))
    if not value in METRIC_TYPES:
      try:
        key_prefix, key_root = key.rsplit(METRIC_DELIM,1)
      except ValueError, e:
        pass
    if not key_root in METRIC_TYPES:
      continue

    key_root, val_type = METRIC_TYPES[key_root]
    key_name = METRIC_DELIM.join([key_prefix, key_root])
    logger('verb', "key_name %s" % (key_name))
    val = collectd.Values(plugin=NAME, type=val_type)
    val.type_instance = key_name
    val.values = [ value ]
    val.dispatch()

# logging function
def logger(t, msg):
    if t == 'err':
        collectd.error('%s: %s' % (NAME, msg))
    elif t == 'warn':
        collectd.warning('%s: %s' % (NAME, msg))
    elif t == 'verb':
        if VERBOSE_LOGGING:
            collectd.info('%s: %s' % (NAME, msg))
    else:
        collectd.notice('%s: %s' % (NAME, msg))
# main
collectd.register_config(configure_callback)
collectd.register_read(read_callback)
<|endoftext|>


<commit_before>import numpy as np
import warnings

import chainer
from chainer import cuda

from chainercv.transforms import center_crop
from chainercv.transforms import resize
from chainercv.transforms import scale
from chainercv.transforms import ten_crop

class FeatureExtractionPredictor(chainer.Chain):

    """Wrapper that adds a prediction method to a feature extraction model.

    The :meth:`predict` takes three steps to make a prediction.

    1. Preprocess input images
    2. Forward the preprocessed images to the network
    3. Average features in the case when more than one crops are extracted.

    Example:

        >>> from chainercv.links import VGG16
        >>> from chainercv.links import FeatureExtractionPredictor
        >>> base_model = VGG16()
        >>> model = FeatureExtractionPredictor(base_model)
        >>> prob = model.predict([img])
        # Predicting multiple features
        >>> model.extractor.feature_names = ['conv5_3', 'fc7']
        >>> conv5_3, fc7 = model.predict([img])

    When :obj:`self.crop == 'center'`, :meth:`predict` extracts features from
    the center crop of the input images.
    When :obj:`self.crop == '10'`, :meth:`predict` extracts features from
    patches that are ten-cropped from the input images.

    When extracting more than one crops from an image, the output of
    :meth:`predict` returns the average of the features computed from the
    crops.

    Args:
        extractor: A feature extraction model. This is a callable chain
            that takes a batch of images and returns a variable or a
            tuple of variables.
        crop_size (int or tuple): The height and the width of an image after
            cropping in preprocessing.
            If this is an integer, the image is cropped to
            :math:`(crop_size, crop_size)`.
        scale_size (int or tuple): If :obj:`scale_size` is an integer, during
            preprocessing, an image is resized so that the length of the
            shorter edge is equal to :obj:`scale_size`. If this is a tuple
            :obj:`(height, width)`, the image is resized to
            :math:`(height, width)`.
        crop ({'center', '10'}): Determines the style of cropping.

    """

    def __init__(self, extractor,
                 crop_size=224, scale_size=256,
                 crop='center'):
        super(FeatureExtractionPredictor, self).__init__()
        self.scale_size = scale_size
        if isinstance(crop_size, int):
            crop_size = (crop_size, crop_size)
        self.crop_size = crop_size
        self.crop = crop

        with self.init_scope():
            self.extractor = extractor

    @property
    def mean(self):
        return self.extractor.mean

    def _prepare(self, img):
        """Prepare an image for feeding it to a model.

        This is a standard preprocessing scheme used by feature extraction
        models.
        First, the image is scaled or resized according to :math:`scale_size`.
        Next, the image is cropped to :math:`crop_size`.
        Last, the image is mean subtracted by an array :obj:`mean`.

        Args:
            img (~numpy.ndarray): An image. This is in CHW and RGB format.
                The range of its value is :math:`[0, 255]`.

        Returns:
            ~numpy.ndarray:
            A preprocessed image. This is 4D array whose batch size is
            the number of crops.

        """
        if isinstance(self.scale_size, int):
            img = scale(img, size=self.scale_size)
        else:
            img = resize(img, size=self.scale_size)

        if self.crop == '10':
            imgs = ten_crop(img, self.crop_size)
        elif self.crop == 'center':
            imgs = center_crop(img, self.crop_size)[np.newaxis]

        imgs -= self.mean[np.newaxis]

        return imgs

    def _average_crops(self, y, n_crop):
        if y.ndim == 4:
            warnings.warn(
                'Four dimensional features are averaged. '
                'If these are batch of 2D spatial features, '
                'their spatial information would be lost.')

        xp = chainer.cuda.get_array_module(y)
        n = y.shape[0] // n_crop
        y = y.reshape((n, n_crop) + y.shape[1:])
        y = xp.sum(y, axis=1) / n_crop
        return y

    def predict(self, imgs):
        """Predict features from images.

        Given :math:`N` input images, this method outputs a batched array with
        batchsize :math:`N`.

        Args:
            imgs (iterable of numpy.ndarray): Array-images.
                All images are in CHW and RGB format
                and the range of their value is :math:`[0, 255]`.

        Returns:
            numpy.ndarray or tuple of numpy.ndarray:
            A batch of features or a tuple of them.

        """
        imgs = self.xp.asarray([self._prepare(img) for img in imgs])
        n_crop = imgs.shape[-4]
        shape = (-1, imgs.shape[-3]) + self.crop_size
        imgs = imgs.reshape(shape)

        with chainer.function.no_backprop_mode():
            imgs = chainer.Variable(imgs)
            activations = self.extractor(imgs)

        if isinstance(activations, tuple):
            output = []
            for activation in activations:
                activation = activation.data
                if n_crop > 1:
                    activation = self._average_crops(activation, n_crop)
                output.append(cuda.to_cpu(activation))
            output = tuple(output)
        else:
            output = cuda.to_cpu(activations.data)
            if n_crop > 1:
                output = self._average_crops(output, n_crop)

        return output
<commit_msg>activations --> features<commit_after>import numpy as np
import warnings

import chainer
from chainer import cuda

from chainercv.transforms import center_crop
from chainercv.transforms import resize
from chainercv.transforms import scale
from chainercv.transforms import ten_crop

class FeatureExtractionPredictor(chainer.Chain):

    """Wrapper that adds a prediction method to a feature extraction model.

    The :meth:`predict` takes three steps to make a prediction.

    1. Preprocess input images
    2. Forward the preprocessed images to the network
    3. Average features in the case when more than one crops are extracted.

    Example:

        >>> from chainercv.links import VGG16
        >>> from chainercv.links import FeatureExtractionPredictor
        >>> base_model = VGG16()
        >>> model = FeatureExtractionPredictor(base_model)
        >>> prob = model.predict([img])
        # Predicting multiple features
        >>> model.extractor.feature_names = ['conv5_3', 'fc7']
        >>> conv5_3, fc7 = model.predict([img])

    When :obj:`self.crop == 'center'`, :meth:`predict` extracts features from
    the center crop of the input images.
    When :obj:`self.crop == '10'`, :meth:`predict` extracts features from
    patches that are ten-cropped from the input images.

    When extracting more than one crops from an image, the output of
    :meth:`predict` returns the average of the features computed from the
    crops.

    Args:
        extractor: A feature extraction model. This is a callable chain
            that takes a batch of images and returns a variable or a
            tuple of variables.
        crop_size (int or tuple): The height and the width of an image after
            cropping in preprocessing.
            If this is an integer, the image is cropped to
            :math:`(crop_size, crop_size)`.
        scale_size (int or tuple): If :obj:`scale_size` is an integer, during
            preprocessing, an image is resized so that the length of the
            shorter edge is equal to :obj:`scale_size`. If this is a tuple
            :obj:`(height, width)`, the image is resized to
            :math:`(height, width)`.
        crop ({'center', '10'}): Determines the style of cropping.

    """

    def __init__(self, extractor,
                 crop_size=224, scale_size=256,
                 crop='center'):
        super(FeatureExtractionPredictor, self).__init__()
        self.scale_size = scale_size
        if isinstance(crop_size, int):
            crop_size = (crop_size, crop_size)
        self.crop_size = crop_size
        self.crop = crop

        with self.init_scope():
            self.extractor = extractor

    @property
    def mean(self):
        return self.extractor.mean

    def _prepare(self, img):
        """Prepare an image for feeding it to a model.

        This is a standard preprocessing scheme used by feature extraction
        models.
        First, the image is scaled or resized according to :math:`scale_size`.
        Next, the image is cropped to :math:`crop_size`.
        Last, the image is mean subtracted by an array :obj:`mean`.

        Args:
            img (~numpy.ndarray): An image. This is in CHW and RGB format.
                The range of its value is :math:`[0, 255]`.

        Returns:
            ~numpy.ndarray:
            A preprocessed image. This is 4D array whose batch size is
            the number of crops.

        """
        if isinstance(self.scale_size, int):
            img = scale(img, size=self.scale_size)
        else:
            img = resize(img, size=self.scale_size)

        if self.crop == '10':
            imgs = ten_crop(img, self.crop_size)
        elif self.crop == 'center':
            imgs = center_crop(img, self.crop_size)[np.newaxis]

        imgs -= self.mean[np.newaxis]

        return imgs

    def _average_crops(self, y, n_crop):
        if y.ndim == 4:
            warnings.warn(
                'Four dimensional features are averaged. '
                'If these are batch of 2D spatial features, '
                'their spatial information would be lost.')

        xp = chainer.cuda.get_array_module(y)
        n = y.shape[0] // n_crop
        y = y.reshape((n, n_crop) + y.shape[1:])
        y = xp.sum(y, axis=1) / n_crop
        return y

    def predict(self, imgs):
        """Predict features from images.

        Given :math:`N` input images, this method outputs a batched array with
        batchsize :math:`N`.

        Args:
            imgs (iterable of numpy.ndarray): Array-images.
                All images are in CHW and RGB format
                and the range of their value is :math:`[0, 255]`.

        Returns:
            numpy.ndarray or tuple of numpy.ndarray:
            A batch of features or a tuple of them.

        """
        imgs = self.xp.asarray([self._prepare(img) for img in imgs])
        n_crop = imgs.shape[-4]
        shape = (-1, imgs.shape[-3]) + self.crop_size
        imgs = imgs.reshape(shape)

        with chainer.function.no_backprop_mode():
            imgs = chainer.Variable(imgs)
            features = self.extractor(imgs)

        if isinstance(features, tuple):
            output = []
            for activation in features:
                activation = activation.data
                if n_crop > 1:
                    activation = self._average_crops(activation, n_crop)
                output.append(cuda.to_cpu(activation))
            output = tuple(output)
        else:
            output = cuda.to_cpu(features.data)
            if n_crop > 1:
                output = self._average_crops(output, n_crop)

        return output
<|endoftext|>


<commit_before># coding=utf-8
# Copyright 2019 The Google Research Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# Lint as: python3
"""Compute realized predictions for a dataset."""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

from absl import app
from absl import flags
from absl import logging

import bert_example
import predict_utils
import tagging_converter
import utils

import tensorflow as tf

FLAGS = flags.FLAGS

flags.DEFINE_string(
    'input_file', None,
    'Path to the input file containing examples for which to compute '
    'predictions.')
flags.DEFINE_enum(
    'input_format', None, ['wikisplit', 'discofuse'],
    'Format which indicates how to parse the input_file.')
flags.DEFINE_string(
    'output_file', None,
    'Path to the TSV file where the predictions are written to.')
flags.DEFINE_string(
    'label_map_file', None,
    'Path to the label map file. Either a JSON file ending with ".json", that '
    'maps each possible tag to an ID, or a text file that has one tag per '
    'line.')
flags.DEFINE_string('vocab_file', None, 'Path to the BERT vocabulary file.')
flags.DEFINE_integer('max_seq_length', 128, 'Maximum sequence length.')
flags.DEFINE_bool(
    'do_lower_case', False,
    'Whether to lower case the input text. Should be True for uncased '
    'models and False for cased models.')
flags.DEFINE_bool('enable_swap_tag', True, 'Whether to enable the SWAP tag.')
flags.DEFINE_string('saved_model', None, 'Path to an exported TF model.')

def main(argv):
  if len(argv) > 1:
    raise app.UsageError('Too many command-line arguments.')
  flags.mark_flag_as_required('input_file')
  flags.mark_flag_as_required('input_format')
  flags.mark_flag_as_required('output_file')
  flags.mark_flag_as_required('label_map_file')
  flags.mark_flag_as_required('vocab_file')
  flags.mark_flag_as_required('saved_model')

  label_map = utils.read_label_map(FLAGS.label_map_file)
  converter = tagging_converter.TaggingConverter(
      tagging_converter.get_phrase_vocabulary_from_label_map(label_map),
      FLAGS.enable_swap_tag)
  builder = bert_example.BertExampleBuilder(label_map, FLAGS.vocab_file,
                                            FLAGS.max_seq_length,
                                            FLAGS.do_lower_case, converter)
  predictor = predict_utils.LaserTaggerPredictor(
      tf.contrib.predictor.from_saved_model(FLAGS.saved_model), builder,
      label_map)

  num_predicted = 0
  with tf.gfile.Open(FLAGS.output_file, 'w') as writer:
    for i, (sources, target) in enumerate(utils.yield_sources_and_targets(
        FLAGS.input_file, FLAGS.input_format)):
      logging.log_every_n(
          logging.INFO,
          f'{i} examples processed, {num_predicted} converted to tf.Example.',
          100)
      prediction = predictor.predict(sources)
      writer.write(f'{" ".join(sources)}\t{prediction}\t{target}\n')
      num_predicted += 1
  logging.info(f'{num_predicted} predictions saved to:\n{FLAGS.output_file}')

if __name__ == '__main__':
  app.run(main)
<commit_msg>Added support for embedding type in prediction<commit_after># coding=utf-8
# Copyright 2019 The Google Research Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# Lint as: python3
"""Compute realized predictions for a dataset."""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

from absl import app
from absl import flags
from absl import logging

import bert_example
import predict_utils
import tagging_converter
import utils

import tensorflow as tf

FLAGS = flags.FLAGS

flags.DEFINE_string(
    'input_file', None,
    'Path to the input file containing examples for which to compute '
    'predictions.')
flags.DEFINE_enum(
    'input_format', None, ['wikisplit', 'discofuse'],
    'Format which indicates how to parse the input_file.')
flags.DEFINE_string(
    'output_file', None,
    'Path to the TSV file where the predictions are written to.')
flags.DEFINE_string(
    'label_map_file', None,
    'Path to the label map file. Either a JSON file ending with ".json", that '
    'maps each possible tag to an ID, or a text file that has one tag per '
    'line.')
flags.DEFINE_string('vocab_file', None, 'Path to the BERT vocabulary file.')
flags.DEFINE_integer('max_seq_length', 128, 'Maximum sequence length.')
flags.DEFINE_bool(
    'do_lower_case', False,
    'Whether to lower case the input text. Should be True for uncased '
    'models and False for cased models.')
flags.DEFINE_bool('enable_swap_tag', True, 'Whether to enable the SWAP tag.')
flags.DEFINE_string('saved_model', None, 'Path to an exported TF model.')
flags.DEFINE_string('embedding_type', None, 'Types of segment id embedding. If '
                    'set to Normal, segment id is all zero. If set to Sentence, '
                    'segment id marks sentences, i.e. 0 for first sentence, 1 for '
                   'second second, 0 for third sentence, etc. If set to POS, '
                   'segment id is the Part of Speech tag of each token.')

def main(argv):
  if len(argv) > 1:
    raise app.UsageError('Too many command-line arguments.')
  flags.mark_flag_as_required('input_file')
  flags.mark_flag_as_required('input_format')
  flags.mark_flag_as_required('output_file')
  flags.mark_flag_as_required('label_map_file')
  flags.mark_flag_as_required('vocab_file')
  flags.mark_flag_as_required('saved_model')
  flags.mark_flag_as_required('embedding_type')

  label_map = utils.read_label_map(FLAGS.label_map_file)
  converter = tagging_converter.TaggingConverter(
      tagging_converter.get_phrase_vocabulary_from_label_map(label_map),
      FLAGS.enable_swap_tag)
  builder = bert_example.BertExampleBuilder(label_map, FLAGS.vocab_file,
                                            FLAGS.max_seq_length,
                                            FLAGS.do_lower_case, converter,
                                            FLAGS.embedding_type)
  predictor = predict_utils.LaserTaggerPredictor(
      tf.contrib.predictor.from_saved_model(FLAGS.saved_model), builder,
      label_map)

  num_predicted = 0
  with tf.gfile.Open(FLAGS.output_file, 'w') as writer:
    for i, (sources, target) in enumerate(utils.yield_sources_and_targets(
        FLAGS.input_file, FLAGS.input_format)):
      logging.log_every_n(
          logging.INFO,
          f'{i} examples processed, {num_predicted} converted to tf.Example.',
          100)
      prediction = predictor.predict(sources)
      writer.write(f'{" ".join(sources)}\t{prediction}\t{target}\n')
      num_predicted += 1
  logging.info(f'{num_predicted} predictions saved to:\n{FLAGS.output_file}')

if __name__ == '__main__':
  app.run(main)
<|endoftext|>