<commit_before># PyVision License
#
# Copyright (c) 2011 David S. Bolme
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
#
# 1. Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
#
# 2. Redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in the
# documentation and/or other materials provided with the distribution.
#
# 3. Neither name of copyright holders nor the names of its contributors
# may be used to endorse or promote products derived from this software
# without specific prior written permission.
#
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR
# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
'''
Created on Jan 11, 2011

@author: bolme
'''

import pyvision as pv
import cv
import numpy as np

HIST_HS = "HIST_HS"
HIST_RGB = "HIST_RGB"

class Histogram:
    '''Basically a wrapper around an OpenCV histogram.'''

    def __init__(self,hist,hist_type,nbins1,nbins2,nbins3):
        '''
        nbins* are the number of bins and should be set to None if they do not exist
        @param hist: the OpenCV histogram
        @param hist_type
        '''
        self.hist = hist
        self.hist_type = hist_type
        self.nbins1 = nbins1
        self.nbins2 = nbins2
        self.nbins3 = nbins3

        assert isinstance(nbins1,int) and nbins1 > 0

    def asMatrix(self):
        if self.nbins2 == None:
            result = np.zeros([self.nbins1])
            for i in range(self.nbins1):
                result[i] = cv.QueryHistValue_1D(self.hist,i)
            return result
        elif self.nbins3 == None:
            result = np.zeros([self.nbins1,self.nbins2])
            for i in range(self.nbins1):
                for j in range(self.nbins2):
                    result[i,j] = cv.QueryHistValue_2D(self.hist,i,j)
            return result
        else:
            result = np.zeros([self.nbins1,self.nbins2,self.nbins3])
            for i in range(self.nbins1):
                for j in range(self.nbins2):
                    for k in range(self.nbins3):
                        result[i,j,k] = cv.QueryHistValue_3D(self.hist,i,j,k)
            return result

    def rescaleMax(self,value=255):
        '''
        Rescale the histogram such that the maximum equals the value.
        '''
        cv.NormalizeHist(self.hist,1)
        _,max_value,_,_ = cv.GetMinMaxHistValue(self.hist)
        if max_value == 0:
            max_value = 1.0
        cv.NormalizeHist(self.hist,255/max_value)

    def rescaleSum(self,value=1.0):
        '''
        Rescale the histogram such that the maximum equals the value.
        '''
        cv.NormalizeHist(self.hist,value)

    def backProject(self,im,bg_hist=None):
        if self.hist_type == pv.HIST_HS:
            return pv.hsBackProjectHist(im,self,bg_hist)
        if self.hist_type == pv.HIST_RGB:
            return pv.rgbBackProjectHist(im,self,bg_hist)
        else:
            raise NotImplementedError("backProject not implemented for type: %s"%self.hist_type)

def hsHist(im,h_bins=32,s_bins=32,mask=None,normalize=True):
    '''
    Compute the hue saturation histogram of an image. (Based on OpenCV example code).

    @param im: the image
    @type im: pv.Image
    @param h_bins: the number of bins for hue.
    @type h_bins: int
    @param s_bins: the number of bins for saturation.
    @type s_bins: int
    @param mask: an image containing a mask
    @type mask: cv.Image or np.array(dtype=np.bool)
    @return: an OpenCV histogram
    @rtype: pv.Histogram
    '''
    w,h = im.size
    hsv = im.asHSV()

    # Extract the H and S planes
    h_plane = cv.CreateImage((w,h), cv.IPL_DEPTH_8U, 1)
    s_plane = cv.CreateImage((w,h), cv.IPL_DEPTH_8U, 1)
    cv.Split(hsv, h_plane, s_plane, None, None)
    planes = [h_plane, s_plane]

    # set the histogram size
    hist_size = [h_bins, s_bins]

    # hue varies from 0 (~0 deg red) to 180 (~360 deg red again */
    h_ranges = [0, 180]

    # saturation varies from 0 (black-gray-white) to
    # 255 (pure spectrum color)
    s_ranges = [0, 255]

    ranges = [h_ranges, s_ranges]

    # Calculate the histogram
    hist = cv.CreateHist(hist_size, cv.CV_HIST_ARRAY, ranges, 1)
    if mask != None:
        mask = mask.asOpenCVBW()
    cv.CalcHist(planes, hist,mask=mask)

    return pv.Histogram(hist,HIST_HS,h_bins,s_bins,None)

def rgbHist(im,r_bins=8,g_bins=8,b_bins=8,mask=None,normalize=True):
    '''
    Compute the hue saturation histogram of an image. (Based on OpenCV example code).

    @param im: the image
    @type im: pv.Image
    @param h_bins: the number of bins for hue.
    @type h_bins: int
    @param s_bins: the number of bins for saturation.
    @type s_bins: int
    @param mask: an image containing a mask
    @type mask: cv.Image or np.array(dtype=np.bool)
    @return: an OpenCV histogram
    @rtype: pv.Histogram
    '''
    w,h = im.size
    bgr = im.asOpenCV()

    # Extract the H and S planes
    b_plane = cv.CreateImage((w,h), cv.IPL_DEPTH_8U, 1)
    g_plane = cv.CreateImage((w,h), cv.IPL_DEPTH_8U, 1)
    r_plane = cv.CreateImage((w,h), cv.IPL_DEPTH_8U, 1)
    cv.Split(bgr, b_plane, g_plane, r_plane, None)
    planes = [b_plane, g_plane, r_plane]

    # set the histogram size
    hist_size = [b_bins, g_bins, r_bins]

    # pixel value ranges
    b_ranges = [0, 255]
    g_ranges = [0, 255]
    r_ranges = [0, 255]

    ranges = [b_ranges, g_ranges, r_ranges]

    # Calculate the histogram
    hist = cv.CreateHist(hist_size, cv.CV_HIST_ARRAY, ranges, 1)
    if mask != None:
        mask = mask.asOpenCVBW()
    cv.CalcHist(planes, hist, mask=mask)

    return pv.Histogram(hist,HIST_RGB,b_bins,g_bins,r_bins)

def hsBackProjectHist(im,fg_hist,bg_hist=None):
    '''
    Compute the hue saturation histogram of an image. (Based on OpenCV example code).

    @param im: the image
    @type im: pv.Image
    @param fg_hist: the histogram
    @type fg_hist: pv.Histogram
    @param bg_hist:
    @type bg_hist: pv.Histogram
    @return: an OpenCV histogram
    @rtype: pv.Image
    '''
    w,h = im.size
    hsv = im.asHSV()

    if bg_hist != None:
        # set the histogram size
        hist_size = [fg_hist.nbins1, fg_hist.nbins2]

        # pixel value ranges
        h_ranges = [0, 180]
        s_ranges = [0, 255]
        ranges = [h_ranges, s_ranges]

        # Calculate the histogram
        prob_hist = cv.CreateHist(hist_size, cv.CV_HIST_ARRAY, ranges, 1)

        fg_hist.rescaleMax(255)
        bg_hist.rescaleMax(255)

        cv.CalcProbDensity(bg_hist.hist, fg_hist.hist, prob_hist, scale=255)

        fg_hist = pv.Histogram(prob_hist,pv.HIST_HS,fg_hist.nbins1, fg_hist.nbins2, None)

    # Extract the H and S planes
    h_plane = cv.CreateImage((w,h), cv.IPL_DEPTH_8U, 1)
    s_plane = cv.CreateImage((w,h), cv.IPL_DEPTH_8U, 1)
    cv.Split(hsv, h_plane, s_plane, None, None)
    planes = [h_plane, s_plane]

    output = cv.CreateImage((w,h), cv.IPL_DEPTH_8U, 1)

    # Normalize the histogram
    fg_hist.rescaleMax(255)

    cv.CalcBackProject(planes,output,fg_hist.hist)

    return pv.Image(output)

def rgbBackProjectHist(im,fg_hist,bg_hist=None):
    '''
    Compute the hue saturation histogram of an image. (Based on OpenCV example code).

    @param im: the image
    @type im: pv.Image
    @param fg_hist: the histogram
    @type fg_hist: pv.Histogram
    @param bg_hist:
    @type bg_hist: pv.Histogram
    @return: an OpenCV histogram
    @rtype: pv.Image
    '''
    w,h = im.size
    bgr = im.asOpenCV()

    if bg_hist != None:
        # set the histogram size
        hist_size = [fg_hist.nbins1, fg_hist.nbins2,fg_hist.nbins3]

        # pixel value ranges
        b_ranges = [0, 255]
        g_ranges = [0, 255]
        r_ranges = [0, 255]
        ranges = [b_ranges, g_ranges, r_ranges]

        # Calculate the histogram
        prob_hist = cv.CreateHist(hist_size, cv.CV_HIST_ARRAY, ranges, 1)

        fg_hist.rescaleMax(255)
        bg_hist.rescaleMax(255)

        cv.CalcProbDensity(bg_hist.hist, fg_hist.hist, prob_hist, scale=255)

        fg_hist = pv.Histogram(prob_hist,pv.HIST_HS,fg_hist.nbins1, fg_hist.nbins2, None)

    # Extract the H and S planes
    b_plane = cv.CreateImage((w,h), cv.IPL_DEPTH_8U, 1)
    g_plane = cv.CreateImage((w,h), cv.IPL_DEPTH_8U, 1)
    r_plane = cv.CreateImage((w,h), cv.IPL_DEPTH_8U, 1)
    cv.Split(bgr, b_plane, g_plane, r_plane, None)
    planes = [b_plane, g_plane, r_plane]

    output = cv.CreateImage((w,h), cv.IPL_DEPTH_8U, 1)

    # Normalize the histogram
    fg_hist.rescaleMax(255)

    cv.CalcBackProject(planes,output,fg_hist.hist)

    return pv.Image(output)

import unittest
import os.path
class ColorTest(unittest.TestCase):

    def test01_hsHist(self):
        '''Test hue saturation histogram.'''
        ilog = pv.ImageLog()
        im = pv.Image(os.path.join(pv.__path__[0],"data","misc","baboon.jpg"))
        mask = np.zeros((512,512),dtype=np.bool)
        mask[150:200,128:300] = True
        m = pv.Image(1.0*mask)
        ilog(im)
        ilog(m)
        hist = hsHist(im,mask=m)
        print hist
        print dir(hist)
        #print dir(hist.bins),hist.bins.channels
        #for i in range(32):
        #    for j in range(30):
        #        print i,j,cv.QueryHistValue_2D(hist,j,i)
        hist.rescaleMax(255)

        print hist.asMatrix()
        #print cv.SetHistBinRanges
        back = hist.backProject(im)
        ilog(back)

        ilog.show()

if __name__ == "__main__":
    #import sys;sys.argv = ['', 'Test.testName']
    unittest.main(testRunner = unittest.TextTestRunner(verbosity=2))<commit_msg>Updated documentation.<commit_after># PyVision License
#
# Copyright (c) 2011 David S. Bolme
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
#
# 1. Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
#
# 2. Redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in the
# documentation and/or other materials provided with the distribution.
#
# 3. Neither name of copyright holders nor the names of its contributors
# may be used to endorse or promote products derived from this software
# without specific prior written permission.
#
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR
# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
'''
Created on Jan 11, 2011

@author: bolme
'''

import pyvision as pv
import cv
import numpy as np

HIST_HS = "HIST_HS"
HIST_RGB = "HIST_RGB"

class Histogram:
    '''Basically a wrapper around an OpenCV histogram.'''

    def __init__(self,hist,hist_type,nbins1,nbins2,nbins3):
        '''
        nbins* are the number of bins and should be set to None if they do not exist
        @param hist: the OpenCV histogram
        @param hist_type
        '''
        self.hist = hist
        self.hist_type = hist_type
        self.nbins1 = nbins1
        self.nbins2 = nbins2
        self.nbins3 = nbins3

        assert isinstance(nbins1,int) and nbins1 > 0

    def asMatrix(self):
        if self.nbins2 == None:
            result = np.zeros([self.nbins1])
            for i in range(self.nbins1):
                result[i] = cv.QueryHistValue_1D(self.hist,i)
            return result
        elif self.nbins3 == None:
            result = np.zeros([self.nbins1,self.nbins2])
            for i in range(self.nbins1):
                for j in range(self.nbins2):
                    result[i,j] = cv.QueryHistValue_2D(self.hist,i,j)
            return result
        else:
            result = np.zeros([self.nbins1,self.nbins2,self.nbins3])
            for i in range(self.nbins1):
                for j in range(self.nbins2):
                    for k in range(self.nbins3):
                        result[i,j,k] = cv.QueryHistValue_3D(self.hist,i,j,k)
            return result

    def rescaleMax(self,value=255):
        '''
        Rescale the histogram such that the maximum equals the value.
        '''
        cv.NormalizeHist(self.hist,1)
        _,max_value,_,_ = cv.GetMinMaxHistValue(self.hist)
        if max_value == 0:
            max_value = 1.0
        cv.NormalizeHist(self.hist,255/max_value)

    def rescaleSum(self,value=1.0):
        '''
        Rescale the histogram such that the maximum equals the value.
        '''
        cv.NormalizeHist(self.hist,value)

    def backProject(self,im,bg_hist=None):
        if self.hist_type == pv.HIST_HS:
            return pv.hsBackProjectHist(im,self,bg_hist)
        if self.hist_type == pv.HIST_RGB:
            return pv.rgbBackProjectHist(im,self,bg_hist)
        else:
            raise NotImplementedError("backProject not implemented for type: %s"%self.hist_type)

def hsHist(im,h_bins=32,s_bins=32,mask=None,normalize=True):
    '''
    Compute the hue saturation histogram of an image. (Based on OpenCV example code).

    @param im: the image
    @type im: pv.Image
    @param h_bins: the number of bins for hue.
    @type h_bins: int
    @param s_bins: the number of bins for saturation.
    @type s_bins: int
    @param mask: an image containing a mask
    @type mask: cv.Image or np.array(dtype=np.bool)
    @return: an OpenCV histogram
    @rtype: pv.Histogram
    '''
    w,h = im.size
    hsv = im.asHSV()

    # Extract the H and S planes
    h_plane = cv.CreateImage((w,h), cv.IPL_DEPTH_8U, 1)
    s_plane = cv.CreateImage((w,h), cv.IPL_DEPTH_8U, 1)
    cv.Split(hsv, h_plane, s_plane, None, None)
    planes = [h_plane, s_plane]

    # set the histogram size
    hist_size = [h_bins, s_bins]

    # hue varies from 0 (~0 deg red) to 180 (~360 deg red again */
    h_ranges = [0, 180]

    # saturation varies from 0 (black-gray-white) to
    # 255 (pure spectrum color)
    s_ranges = [0, 255]

    ranges = [h_ranges, s_ranges]

    # Calculate the histogram
    hist = cv.CreateHist(hist_size, cv.CV_HIST_ARRAY, ranges, 1)
    if mask != None:
        mask = mask.asOpenCVBW()
    cv.CalcHist(planes, hist,mask=mask)

    return pv.Histogram(hist,HIST_HS,h_bins,s_bins,None)

def rgbHist(im,r_bins=8,g_bins=8,b_bins=8,mask=None,normalize=True):
    '''
    Compute the rgb saturation histogram of an image. (Based on OpenCV example code).

    @param im: the image
    @type im: pv.Image
    @param r_bins: the number of bins for hue.
    @type r_bins: int
    @param g_bins: the number of bins for hue.
    @type g_bins: int
    @param b_bins: the number of bins for hue.
    @type b_bins: int
    @param mask: an image containing a mask
    @type mask: cv.Image or np.array(dtype=np.bool)
    @return: an OpenCV histogram
    @rtype: pv.Histogram
    '''
    w,h = im.size
    bgr = im.asOpenCV()

    # Extract the H and S planes
    b_plane = cv.CreateImage((w,h), cv.IPL_DEPTH_8U, 1)
    g_plane = cv.CreateImage((w,h), cv.IPL_DEPTH_8U, 1)
    r_plane = cv.CreateImage((w,h), cv.IPL_DEPTH_8U, 1)
    cv.Split(bgr, b_plane, g_plane, r_plane, None)
    planes = [b_plane, g_plane, r_plane]

    # set the histogram size
    hist_size = [b_bins, g_bins, r_bins]

    # pixel value ranges
    b_ranges = [0, 255]
    g_ranges = [0, 255]
    r_ranges = [0, 255]

    ranges = [b_ranges, g_ranges, r_ranges]

    # Calculate the histogram
    hist = cv.CreateHist(hist_size, cv.CV_HIST_ARRAY, ranges, 1)
    if mask != None:
        mask = mask.asOpenCVBW()
    cv.CalcHist(planes, hist, mask=mask)

    return pv.Histogram(hist,HIST_RGB,b_bins,g_bins,r_bins)

def hsBackProjectHist(im,fg_hist,bg_hist=None):
    '''
    Compute the hue saturation histogram of an image. (Based on OpenCV example code).

    @param im: the image
    @type im: pv.Image
    @param fg_hist: the histogram
    @type fg_hist: pv.Histogram
    @param bg_hist:
    @type bg_hist: pv.Histogram
    @return: an OpenCV histogram
    @rtype: pv.Image
    '''
    w,h = im.size
    hsv = im.asHSV()

    if bg_hist != None:
        # set the histogram size
        hist_size = [fg_hist.nbins1, fg_hist.nbins2]

        # pixel value ranges
        h_ranges = [0, 180]
        s_ranges = [0, 255]
        ranges = [h_ranges, s_ranges]

        # Calculate the histogram
        prob_hist = cv.CreateHist(hist_size, cv.CV_HIST_ARRAY, ranges, 1)

        fg_hist.rescaleMax(255)
        bg_hist.rescaleMax(255)

        cv.CalcProbDensity(bg_hist.hist, fg_hist.hist, prob_hist, scale=255)

        fg_hist = pv.Histogram(prob_hist,pv.HIST_HS,fg_hist.nbins1, fg_hist.nbins2, None)

    # Extract the H and S planes
    h_plane = cv.CreateImage((w,h), cv.IPL_DEPTH_8U, 1)
    s_plane = cv.CreateImage((w,h), cv.IPL_DEPTH_8U, 1)
    cv.Split(hsv, h_plane, s_plane, None, None)
    planes = [h_plane, s_plane]

    output = cv.CreateImage((w,h), cv.IPL_DEPTH_8U, 1)

    # Normalize the histogram
    fg_hist.rescaleMax(255)

    cv.CalcBackProject(planes,output,fg_hist.hist)

    return pv.Image(output)

def rgbBackProjectHist(im,fg_hist,bg_hist=None):
    '''
    Compute the hue saturation histogram of an image. (Based on OpenCV example code).

    @param im: the image
    @type im: pv.Image
    @param fg_hist: the histogram
    @type fg_hist: pv.Histogram
    @param bg_hist:
    @type bg_hist: pv.Histogram
    @return: an OpenCV histogram
    @rtype: pv.Image
    '''
    w,h = im.size
    bgr = im.asOpenCV()

    if bg_hist != None:
        # set the histogram size
        hist_size = [fg_hist.nbins1, fg_hist.nbins2,fg_hist.nbins3]

        # pixel value ranges
        b_ranges = [0, 255]
        g_ranges = [0, 255]
        r_ranges = [0, 255]
        ranges = [b_ranges, g_ranges, r_ranges]

        # Calculate the histogram
        prob_hist = cv.CreateHist(hist_size, cv.CV_HIST_ARRAY, ranges, 1)

        fg_hist.rescaleMax(255)
        bg_hist.rescaleMax(255)

        cv.CalcProbDensity(bg_hist.hist, fg_hist.hist, prob_hist, scale=255)

        fg_hist = pv.Histogram(prob_hist,pv.HIST_HS,fg_hist.nbins1, fg_hist.nbins2, None)

    # Extract the H and S planes
    b_plane = cv.CreateImage((w,h), cv.IPL_DEPTH_8U, 1)
    g_plane = cv.CreateImage((w,h), cv.IPL_DEPTH_8U, 1)
    r_plane = cv.CreateImage((w,h), cv.IPL_DEPTH_8U, 1)
    cv.Split(bgr, b_plane, g_plane, r_plane, None)
    planes = [b_plane, g_plane, r_plane]

    output = cv.CreateImage((w,h), cv.IPL_DEPTH_8U, 1)

    # Normalize the histogram
    fg_hist.rescaleMax(255)

    cv.CalcBackProject(planes,output,fg_hist.hist)

    return pv.Image(output)

import unittest
import os.path
class ColorTest(unittest.TestCase):

    def test01_hsHist(self):
        '''Test hue saturation histogram.'''
        ilog = pv.ImageLog()
        im = pv.Image(os.path.join(pv.__path__[0],"data","misc","baboon.jpg"))
        mask = np.zeros((512,512),dtype=np.bool)
        mask[150:200,128:300] = True
        m = pv.Image(1.0*mask)
        ilog(im)
        ilog(m)
        hist = hsHist(im,mask=m)
        print hist
        print dir(hist)
        #print dir(hist.bins),hist.bins.channels
        #for i in range(32):
        #    for j in range(30):
        #        print i,j,cv.QueryHistValue_2D(hist,j,i)
        hist.rescaleMax(255)

        print hist.asMatrix()
        #print cv.SetHistBinRanges
        back = hist.backProject(im)
        ilog(back)

        ilog.show()

if __name__ == "__main__":
    #import sys;sys.argv = ['', 'Test.testName']
    unittest.main(testRunner = unittest.TextTestRunner(verbosity=2))<|endoftext|>


<commit_before>from collections import defaultdict
from typing import Dict, Tuple

import torch
from torch import Tensor
from torch.utils.data import DataLoader

from avalanche.training.plugins.strategy_plugin import StrategyPlugin
from avalanche.training.utils import copy_params_dict, zerolike_params_dict

class EWCPlugin(StrategyPlugin):
    """
    Elastic Weight Consolidation (EWC) plugin.
    EWC computes importance of each weight at the end of training on current
    experience. During training on each minibatch, the loss is augmented
    with a penalty which keeps the value of the current weights close to the
    value they had on previous experiences in proportion to their importance
    on that experience. Importances are computed with an additional pass on the
    training set. This plugin does not use task identities.
    """

    def __init__(self, ewc_lambda, mode='separate', decay_factor=None,
                 keep_importance_data=False):
        """
        :param ewc_lambda: hyperparameter to weigh the penalty inside the total
               loss. The larger the lambda, the larger the regularization.
        :param mode: `separate` to keep a separate penalty for each previous
               experience.
               `online` to keep a single penalty summed with a decay factor
               over all previous tasks.
        :param decay_factor: used only if mode is `online`.
               It specifies the decay term of the importance matrix.
        :param keep_importance_data: if True, keep in memory both parameter
                values and importances for all previous task, for all modes.
                If False, keep only last parameter values and importances.
                If mode is `separate`, the value of `keep_importance_data` is
                set to be True.
        """

        super().__init__()

        assert mode == 'separate' or mode == 'online', \
            'Mode must be separate or online.'

        self.ewc_lambda = ewc_lambda
        self.mode = mode
        self.decay_factor = decay_factor

        if self.mode == 'separate':
            self.keep_importance_data = True
        else:
            self.keep_importance_data = keep_importance_data

        self.saved_params = defaultdict(list)
        self.importances = defaultdict(list)

    def before_backward(self, strategy, **kwargs):
        """
        Compute EWC penalty and add it to the loss.
        """

        if strategy.training_exp_counter == 0:
            return

        penalty = torch.tensor(0).float().to(strategy.device)

        if self.mode == 'separate':
            for experience in range(strategy.training_exp_counter):
                for (_, cur_param), (_, saved_param), (_, imp) in zip(
                        strategy.model.named_parameters(),
                        self.saved_params[experience],
                        self.importances[experience]):
                    penalty += (imp * (cur_param - saved_param).pow(2)).sum()
        elif self.mode == 'online':
            for (_, cur_param), (_, saved_param), (_, imp) in zip(
                    strategy.model.named_parameters(),
                    self.saved_params[strategy.training_exp_counter],
                    self.importances[strategy.training_exp_counter]):
                penalty += (imp * (cur_param - saved_param).pow(2)).sum()
        else:
            raise ValueError('Wrong EWC mode.')

        strategy.loss += self.ewc_lambda * penalty

    def after_training_exp(self, strategy, **kwargs):
        """
        Compute importances of parameters after each experience.
        """

        importances = self.compute_importances(strategy.model,
                                               strategy.criterion,
                                               strategy.optimizer,
                                               strategy.experience.dataset,
                                               strategy.device,
                                               strategy.train_mb_size)
        self.update_importances(importances, strategy.training_exp_counter)
        self.saved_params[strategy.training_exp_counter] = \
            copy_params_dict(strategy.model)
        # clear previuos parameter values
        if strategy.training_exp_counter > 0 and \
                (not self.keep_importance_data):
            del self.saved_params[strategy.training_exp_counter - 1]

    def compute_importances(self, model, criterion, optimizer,
                            dataset, device, batch_size):
        """
        Compute EWC importance matrix for each parameter
        """

        model.train()

        # list of list
        importances = zerolike_params_dict(model)
        dataloader = DataLoader(dataset, batch_size=batch_size)
        for i, (x, y, _) in enumerate(dataloader):
            x, y = x.to(device), y.to(device)

            optimizer.zero_grad()
            out = model(x)
            loss = criterion(out, y)
            loss.backward()

            for (k1, p), (k2, imp) in zip(model.named_parameters(),
                                          importances):
                assert (k1 == k2)
                imp += p.grad.data.clone().pow(2)

        # average over mini batch length
        for _, imp in importances:
            imp /= float(len(dataloader))

        return importances

    @torch.no_grad()
    def update_importances(self, importances, t):
        """
        Update importance for each parameter based on the currently computed
        importances.
        """

        if self.mode == 'separate' or t == 0:
            self.importances[t] = importances
        elif self.mode == 'online':
            for (k1, old_imp), (k2, curr_imp) in \
                    zip(self.importances[t - 1], importances):
                assert k1 == k2, 'Error in importance computation.'
                self.importances[t].append(
                    (k1, (self.decay_factor * old_imp + curr_imp)))

            # clear previous parameter importances
            if t > 0 and (not self.keep_importance_data):
                del self.importances[t - 1]

        else:
            raise ValueError("Wrong EWC mode.")

ParamDict = Dict[str, Tensor]
EwcDataType = Tuple[ParamDict, ParamDict]
<commit_msg>Ignored parameters without gradient in importance<commit_after>from collections import defaultdict
from typing import Dict, Tuple

import torch
from torch import Tensor
from torch.utils.data import DataLoader

from avalanche.training.plugins.strategy_plugin import StrategyPlugin
from avalanche.training.utils import copy_params_dict, zerolike_params_dict

class EWCPlugin(StrategyPlugin):
    """
    Elastic Weight Consolidation (EWC) plugin.
    EWC computes importance of each weight at the end of training on current
    experience. During training on each minibatch, the loss is augmented
    with a penalty which keeps the value of the current weights close to the
    value they had on previous experiences in proportion to their importance
    on that experience. Importances are computed with an additional pass on the
    training set. This plugin does not use task identities.
    """

    def __init__(self, ewc_lambda, mode='separate', decay_factor=None,
                 keep_importance_data=False):
        """
        :param ewc_lambda: hyperparameter to weigh the penalty inside the total
               loss. The larger the lambda, the larger the regularization.
        :param mode: `separate` to keep a separate penalty for each previous
               experience.
               `online` to keep a single penalty summed with a decay factor
               over all previous tasks.
        :param decay_factor: used only if mode is `online`.
               It specifies the decay term of the importance matrix.
        :param keep_importance_data: if True, keep in memory both parameter
                values and importances for all previous task, for all modes.
                If False, keep only last parameter values and importances.
                If mode is `separate`, the value of `keep_importance_data` is
                set to be True.
        """

        super().__init__()

        assert mode == 'separate' or mode == 'online', \
            'Mode must be separate or online.'

        self.ewc_lambda = ewc_lambda
        self.mode = mode
        self.decay_factor = decay_factor

        if self.mode == 'separate':
            self.keep_importance_data = True
        else:
            self.keep_importance_data = keep_importance_data

        self.saved_params = defaultdict(list)
        self.importances = defaultdict(list)

    def before_backward(self, strategy, **kwargs):
        """
        Compute EWC penalty and add it to the loss.
        """

        if strategy.training_exp_counter == 0:
            return

        penalty = torch.tensor(0).float().to(strategy.device)

        if self.mode == 'separate':
            for experience in range(strategy.training_exp_counter):
                for (_, cur_param), (_, saved_param), (_, imp) in zip(
                        strategy.model.named_parameters(),
                        self.saved_params[experience],
                        self.importances[experience]):
                    penalty += (imp * (cur_param - saved_param).pow(2)).sum()
        elif self.mode == 'online':
            for (_, cur_param), (_, saved_param), (_, imp) in zip(
                    strategy.model.named_parameters(),
                    self.saved_params[strategy.training_exp_counter],
                    self.importances[strategy.training_exp_counter]):
                penalty += (imp * (cur_param - saved_param).pow(2)).sum()
        else:
            raise ValueError('Wrong EWC mode.')

        strategy.loss += self.ewc_lambda * penalty

    def after_training_exp(self, strategy, **kwargs):
        """
        Compute importances of parameters after each experience.
        """

        importances = self.compute_importances(strategy.model,
                                               strategy.criterion,
                                               strategy.optimizer,
                                               strategy.experience.dataset,
                                               strategy.device,
                                               strategy.train_mb_size)
        self.update_importances(importances, strategy.training_exp_counter)
        self.saved_params[strategy.training_exp_counter] = \
            copy_params_dict(strategy.model)
        # clear previuos parameter values
        if strategy.training_exp_counter > 0 and \
                (not self.keep_importance_data):
            del self.saved_params[strategy.training_exp_counter - 1]

    def compute_importances(self, model, criterion, optimizer,
                            dataset, device, batch_size):
        """
        Compute EWC importance matrix for each parameter
        """

        model.train()

        # list of list
        importances = zerolike_params_dict(model)
        dataloader = DataLoader(dataset, batch_size=batch_size)
        for i, (x, y, _) in enumerate(dataloader):
            x, y = x.to(device), y.to(device)

            optimizer.zero_grad()
            out = model(x)
            loss = criterion(out, y)
            loss.backward()

            for (k1, p), (k2, imp) in zip(model.named_parameters(),
                                          importances):
                assert (k1 == k2)
                if p.grad is not None:
                    imp += p.grad.data.clone().pow(2)

        # average over mini batch length
        for _, imp in importances:
            imp /= float(len(dataloader))

        return importances

    @torch.no_grad()
    def update_importances(self, importances, t):
        """
        Update importance for each parameter based on the currently computed
        importances.
        """

        if self.mode == 'separate' or t == 0:
            self.importances[t] = importances
        elif self.mode == 'online':
            for (k1, old_imp), (k2, curr_imp) in \
                    zip(self.importances[t - 1], importances):
                assert k1 == k2, 'Error in importance computation.'
                self.importances[t].append(
                    (k1, (self.decay_factor * old_imp + curr_imp)))

            # clear previous parameter importances
            if t > 0 and (not self.keep_importance_data):
                del self.importances[t - 1]

        else:
            raise ValueError("Wrong EWC mode.")

ParamDict = Dict[str, Tensor]
EwcDataType = Tuple[ParamDict, ParamDict]
<|endoftext|>


<commit_before>#! /usr/bin/env python
# -*- coding:utf-8 -*-

#
# Pose estimation of the camera
#

# External dependencies
import pickle
import cv2
import numpy as np

# Calibration pattern size
pattern_size = ( 9, 6 )
# Load calibration file
with open( 'calibration.pkl', 'rb' ) as calibration_file :
    calibration = pickle.load( calibration_file )
# Get the camera
camera = cv2.VideoCapture( 0 )
# Change the camera resolution
#camera.set( cv2.cv.CV_CAP_PROP_FRAME_WIDTH, 1920 )
#camera.set( cv2.cv.CV_CAP_PROP_FRAME_HEIGHT, 1200 )
# Acquisition loop
while( True ) :
    # Capture image-by-image
    _, image = camera.read()
    # Copy the image for display
    chessboard = np.copy( image )
    # Find the chessboard corners on the image
    found, corners = cv2.findChessboardCorners( chessboard, pattern_size, flags = cv2.CALIB_CB_FAST_CHECK )
    # Draw the chessboard corners on the image
    if found : cv2.drawChessboardCorners( chessboard, pattern_size, corners, found )
    # Display the resulting image
    cv2.imshow( 'USB Camera', chessboard )
    # Keyboard interruption
    key = cv2.waitKey( 1 ) & 0xFF
    # Escape : quit the application
    if key == 27 : break
    # Space : save the image and quit
    elif key == 32 :
        cv2.imwrite( 'pose-estimation.png', image )
        break
# Release the camera
camera.release()
# Close OpenCV windows
cv2.destroyAllWindows()
# Chessboard pattern
pattern_points = np.zeros( ( np.prod( pattern_size ), 3 ), np.float32 )
pattern_points[ :, :2 ] = np.indices( pattern_size ).T.reshape( -1, 2 )
# 3D points
object_points = []
# 2D points
image_points = []
# Convert the image in grayscale
image = cv2.cvtColor( image, cv2.COLOR_BGR2GRAY )
# Chessboard detection flags
flags  = 0
flags |= cv2.CALIB_CB_ADAPTIVE_THRESH
flags |= cv2.CALIB_CB_NORMALIZE_IMAGE
# Find the chessboard corners on the image
found, corners = cv2.findChessboardCorners( image, pattern_size, flags = flags )
# Pattern not found
if not found : print( 'Chessboard not found...' )
# Termination criteria for the corner detection
criteria = ( cv2.TERM_CRITERIA_MAX_ITER + cv2.TERM_CRITERIA_EPS, 30, 1e-5 )
# Refine the corner positions
cv2.cornerSubPix( image, corners, ( 11, 11 ), ( -1, -1 ), criteria )
# Pose estimation flags
flags = 0
#flags = CV_ITERATIVE
#flags = CV_P3P
#flags = CV_EPNP
# Solve the pose
ret, rvec, tvec = cv2.solvePnP( pattern_points, corners.reshape( -1, 2 ), calibration['camera_matrix'], calibration['dist_coefs'] ) #,
#    calibration['rvecs'], calibration['tvecs'], True, flags )
print ret
print rvec
print tvec
<commit_msg>Remove acquisition. Add chesbboard square size, image undistortion, and pose estimation.<commit_after>#! /usr/bin/env python
# -*- coding:utf-8 -*-

#
# Pose estimation of the camera
#

# External dependencies
import pickle
import cv2
import numpy as np

# Calibration pattern size
pattern_size = ( 9, 6 )
# Chessboard pattern
pattern_points = np.zeros( ( np.prod( pattern_size ), 3 ), np.float32 )
pattern_points[ :, :2 ] = np.indices( pattern_size ).T.reshape( -1, 2 )
# Chessboard square size
pattern_points *= 34.15
# 3D points
object_points = []
# 2D points
image_points = []
# Load calibration file
with open( 'calibration.pkl', 'rb' ) as calibration_file :
    calibration = pickle.load( calibration_file )
# Read the image
image = cv2.imread( 'pose-estimation.png' )
# Get image size
image_size = image.shape[ :2 ][ ::-1 ]
# Compute new optimal camera matrix
new_camera_matrix, roi = cv2.getOptimalNewCameraMatrix( calibration['camera_matrix'], calibration['dist_coefs'], image_size, 1, image_size )
# Remove lens distortion
rectified_image = cv2.undistort( image, calibration['camera_matrix'], calibration['dist_coefs'], None, new_camera_matrix )
print calibration['camera_matrix']
print new_camera_matrix
print roi
# Print ROI
#cv2.rectangle( rectified_image, roi[:2], roi[2:], (0,0,255), 2 )
#cv2.imshow( 'rectified', rectified_image )
#cv2.waitKey()
# Convert the image in grayscale
rectified_image = cv2.cvtColor( rectified_image, cv2.COLOR_BGR2GRAY )
# Chessboard detection flags
flags  = 0
flags |= cv2.CALIB_CB_ADAPTIVE_THRESH
flags |= cv2.CALIB_CB_NORMALIZE_IMAGE
# Find the chessboard corners on the image
found, corners = cv2.findChessboardCorners( rectified_image, pattern_size, flags = flags )
# Pattern not found
if not found : print( 'Chessboard not found...' )
# Termination criteria for the corner detection
criteria = ( cv2.TERM_CRITERIA_MAX_ITER + cv2.TERM_CRITERIA_EPS, 30, 1e-5 )
# Refine the corner positions
cv2.cornerSubPix( rectified_image, corners, ( 11, 11 ), ( -1, -1 ), criteria )
# Pose estimation flags
flags = 0
#flags = CV_ITERATIVE
#flags = CV_P3P
#flags = CV_EPNP
# Solve the pose
_, rotation_vector, translation_vector = cv2.solvePnP( pattern_points, corners.reshape( -1, 2 ), calibration['camera_matrix'], calibration['dist_coefs'] )
print rotation_vector
print translation_vector
rotation_matrix, _ = cv2.Rodrigues( rotation_vector )
print rotation_matrix
<|endoftext|>


<commit_before>from collections import defaultdict
from typing import Dict, Tuple

import torch
from torch import Tensor
from torch.utils.data import DataLoader

from avalanche.models.utils import avalanche_forward
from avalanche.training.plugins.strategy_plugin import StrategyPlugin
from avalanche.training.utils import copy_params_dict, zerolike_params_dict

class EWCPlugin(StrategyPlugin):
    """
    Elastic Weight Consolidation (EWC) plugin.
    EWC computes importance of each weight at the end of training on current
    experience. During training on each minibatch, the loss is augmented
    with a penalty which keeps the value of the current weights close to the
    value they had on previous experiences in proportion to their importance
    on that experience. Importances are computed with an additional pass on the
    training set. This plugin does not use task identities.
    """

    def __init__(self, ewc_lambda, mode='separate', decay_factor=None,
                 keep_importance_data=False):
        """
        :param ewc_lambda: hyperparameter to weigh the penalty inside the total
               loss. The larger the lambda, the larger the regularization.
        :param mode: `separate` to keep a separate penalty for each previous
               experience.
               `online` to keep a single penalty summed with a decay factor
               over all previous tasks.
        :param decay_factor: used only if mode is `online`.
               It specifies the decay term of the importance matrix.
        :param keep_importance_data: if True, keep in memory both parameter
                values and importances for all previous task, for all modes.
                If False, keep only last parameter values and importances.
                If mode is `separate`, the value of `keep_importance_data` is
                set to be True.
        """

        super().__init__()

        assert mode == 'separate' or mode == 'online', \
            'Mode must be separate or online.'

        self.ewc_lambda = ewc_lambda
        self.mode = mode
        self.decay_factor = decay_factor

        if self.mode == 'separate':
            self.keep_importance_data = True
        else:
            self.keep_importance_data = keep_importance_data

        self.saved_params = defaultdict(list)
        self.importances = defaultdict(list)

    def before_backward(self, strategy, **kwargs):
        """
        Compute EWC penalty and add it to the loss.
        """

        if strategy.training_exp_counter == 0:
            return

        penalty = torch.tensor(0).float().to(strategy.device)

        if self.mode == 'separate':
            for experience in range(strategy.training_exp_counter):
                for (_, cur_param), (_, saved_param), (_, imp) in zip(
                        strategy.model.named_parameters(),
                        self.saved_params[experience],
                        self.importances[experience]):
                    penalty += (imp * (cur_param - saved_param).pow(2)).sum()
        elif self.mode == 'online':
            for (_, cur_param), (_, saved_param), (_, imp) in zip(
                    strategy.model.named_parameters(),
                    self.saved_params[strategy.training_exp_counter],
                    self.importances[strategy.training_exp_counter]):
                penalty += (imp * (cur_param - saved_param).pow(2)).sum()
        else:
            raise ValueError('Wrong EWC mode.')

        strategy.loss += self.ewc_lambda * penalty

    def after_training_exp(self, strategy, **kwargs):
        """
        Compute importances of parameters after each experience.
        """

        importances = self.compute_importances(strategy.model,
                                               strategy._criterion,
                                               strategy.optimizer,
                                               strategy.experience.dataset,
                                               strategy.device,
                                               strategy.train_mb_size)
        self.update_importances(importances, strategy.training_exp_counter)
        self.saved_params[strategy.training_exp_counter] = \
            copy_params_dict(strategy.model)
        # clear previuos parameter values
        if strategy.training_exp_counter > 0 and \
                (not self.keep_importance_data):
            del self.saved_params[strategy.training_exp_counter - 1]

    def compute_importances(self, model, criterion, optimizer,
                            dataset, device, batch_size):
        """
        Compute EWC importance matrix for each parameter
        """

        model.eval()

        # list of list
        importances = zerolike_params_dict(model)
        dataloader = DataLoader(dataset, batch_size=batch_size)
        for i, (x, y, task_labels) in enumerate(dataloader):
            x, y = x.to(device), y.to(device)

            optimizer.zero_grad()
            out = avalanche_forward(model, x, task_labels)
            loss = criterion(out, y)
            loss.backward()

            for (k1, p), (k2, imp) in zip(model.named_parameters(),
                                          importances):
                assert (k1 == k2)
                if p.grad is not None:
                    imp += p.grad.data.clone().pow(2)

        # average over mini batch length
        for _, imp in importances:
            imp /= float(len(dataloader))

        return importances

    @torch.no_grad()
    def update_importances(self, importances, t):
        """
        Update importance for each parameter based on the currently computed
        importances.
        """

        if self.mode == 'separate' or t == 0:
            self.importances[t] = importances
        elif self.mode == 'online':
            for (k1, old_imp), (k2, curr_imp) in \
                    zip(self.importances[t - 1], importances):
                assert k1 == k2, 'Error in importance computation.'
                self.importances[t].append(
                    (k1, (self.decay_factor * old_imp + curr_imp)))

            # clear previous parameter importances
            if t > 0 and (not self.keep_importance_data):
                del self.importances[t - 1]

        else:
            raise ValueError("Wrong EWC mode.")

ParamDict = Dict[str, Tensor]
EwcDataType = Tuple[ParamDict, ParamDict]
<commit_msg>Fix online EWC penalty calculation<commit_after>from collections import defaultdict
from typing import Dict, Tuple

import torch
from torch import Tensor
from torch.utils.data import DataLoader

from avalanche.models.utils import avalanche_forward
from avalanche.training.plugins.strategy_plugin import StrategyPlugin
from avalanche.training.utils import copy_params_dict, zerolike_params_dict

class EWCPlugin(StrategyPlugin):
    """
    Elastic Weight Consolidation (EWC) plugin.
    EWC computes importance of each weight at the end of training on current
    experience. During training on each minibatch, the loss is augmented
    with a penalty which keeps the value of the current weights close to the
    value they had on previous experiences in proportion to their importance
    on that experience. Importances are computed with an additional pass on the
    training set. This plugin does not use task identities.
    """

    def __init__(self, ewc_lambda, mode='separate', decay_factor=None,
                 keep_importance_data=False):
        """
        :param ewc_lambda: hyperparameter to weigh the penalty inside the total
               loss. The larger the lambda, the larger the regularization.
        :param mode: `separate` to keep a separate penalty for each previous
               experience.
               `online` to keep a single penalty summed with a decay factor
               over all previous tasks.
        :param decay_factor: used only if mode is `online`.
               It specifies the decay term of the importance matrix.
        :param keep_importance_data: if True, keep in memory both parameter
                values and importances for all previous task, for all modes.
                If False, keep only last parameter values and importances.
                If mode is `separate`, the value of `keep_importance_data` is
                set to be True.
        """

        super().__init__()

        assert mode == 'separate' or mode == 'online', \
            'Mode must be separate or online.'

        self.ewc_lambda = ewc_lambda
        self.mode = mode
        self.decay_factor = decay_factor

        if self.mode == 'separate':
            self.keep_importance_data = True
        else:
            self.keep_importance_data = keep_importance_data

        self.saved_params = defaultdict(list)
        self.importances = defaultdict(list)

    def before_backward(self, strategy, **kwargs):
        """
        Compute EWC penalty and add it to the loss.
        """

        if strategy.training_exp_counter == 0:
            return

        penalty = torch.tensor(0).float().to(strategy.device)

        if self.mode == 'separate':
            for experience in range(strategy.training_exp_counter):
                for (_, cur_param), (_, saved_param), (_, imp) in zip(
                        strategy.model.named_parameters(),
                        self.saved_params[experience],
                        self.importances[experience]):
                    penalty += (imp * (cur_param - saved_param).pow(2)).sum()
        elif self.mode == 'online':
            prev_exp = strategy.training_exp_counter - 1
            for (_, cur_param), (_, saved_param), (_, imp) in zip(
                    strategy.model.named_parameters(),
                    self.saved_params[prev_exp],
                    self.importances[prev_exp]):
                penalty += (imp * (cur_param - saved_param).pow(2)).sum()
        else:
            raise ValueError('Wrong EWC mode.')

        strategy.loss += self.ewc_lambda * penalty

    def after_training_exp(self, strategy, **kwargs):
        """
        Compute importances of parameters after each experience.
        """

        importances = self.compute_importances(strategy.model,
                                               strategy._criterion,
                                               strategy.optimizer,
                                               strategy.experience.dataset,
                                               strategy.device,
                                               strategy.train_mb_size)
        self.update_importances(importances, strategy.training_exp_counter)
        self.saved_params[strategy.training_exp_counter] = \
            copy_params_dict(strategy.model)
        # clear previous parameter values
        if strategy.training_exp_counter > 0 and \
                (not self.keep_importance_data):
            del self.saved_params[strategy.training_exp_counter - 1]

    def compute_importances(self, model, criterion, optimizer,
                            dataset, device, batch_size):
        """
        Compute EWC importance matrix for each parameter
        """

        model.eval()

        # list of list
        importances = zerolike_params_dict(model)
        dataloader = DataLoader(dataset, batch_size=batch_size)
        for i, (x, y, task_labels) in enumerate(dataloader):
            x, y = x.to(device), y.to(device)

            optimizer.zero_grad()
            out = avalanche_forward(model, x, task_labels)
            loss = criterion(out, y)
            loss.backward()

            for (k1, p), (k2, imp) in zip(model.named_parameters(),
                                          importances):
                assert (k1 == k2)
                if p.grad is not None:
                    imp += p.grad.data.clone().pow(2)

        # average over mini batch length
        for _, imp in importances:
            imp /= float(len(dataloader))

        return importances

    @torch.no_grad()
    def update_importances(self, importances, t):
        """
        Update importance for each parameter based on the currently computed
        importances.
        """

        if self.mode == 'separate' or t == 0:
            self.importances[t] = importances
        elif self.mode == 'online':
            for (k1, old_imp), (k2, curr_imp) in \
                    zip(self.importances[t - 1], importances):
                assert k1 == k2, 'Error in importance computation.'
                self.importances[t].append(
                    (k1, (self.decay_factor * old_imp + curr_imp)))

            # clear previous parameter importances
            if t > 0 and (not self.keep_importance_data):
                del self.importances[t - 1]

        else:
            raise ValueError("Wrong EWC mode.")

ParamDict = Dict[str, Tensor]
EwcDataType = Tuple[ParamDict, ParamDict]
<|endoftext|>


<commit_before>import re
from urllib import parse
from pupa.scrape import Scraper, Person
from .base import Spatula, Page

class SenDetail(Page):
    list_xpath = '//h4[contains(text(), "Office")]'

    def process_list_item(self, office):
        (name, ) = office.xpath('text()')
        if name == "Tallahassee Office":
            type_ = 'capitol'
        else:
            type_ = 'district'

        address_lines = [x.strip() for x in
                         office.xpath('following-sibling::div[1]')[0].text_content().splitlines()
                         if x.strip()]

        clean_address_lines = []
        fax = phone = None
        PHONE_RE = r'\(\d{3}\)\s\d{3}\-\d{4}'
        after_phone = False

        for line in address_lines:
            if re.search(r'(?i)open\s+\w+day', address_lines[0]):
                continue
            elif 'FAX' in line:
                fax = line.replace('FAX ', '')
                after_phone = True
            elif re.search(PHONE_RE, line):
                phone = line
                after_phone = True
            elif not after_phone:
                clean_address_lines.append(line)

        if phone:
            self.obj.add_contact_detail(type='voice', value=phone, note=type_)
        if fax:
            self.obj.add_contact_detail(type='fax', value=fax, note=type_)

        # address
        address = "\n".join(clean_address_lines)
        address = re.sub(r'\s{2,}', " ", address)
        if address:
            self.obj.add_contact_detail(type='address', value=address, note=type_)

    def process_page(self):
        super().process_page()
        email = self.doc.xpath('//a[contains(@href, "mailto:")]')[0].get('href').split(':')[-1]
        self.obj.add_contact_detail(type='email', value=email)

        self.obj.image = self.doc.xpath('//div[@id="sidebar"]//img/@src').pop()

class SenList(Page):
    url = "http://www.flsenate.gov/Senators/"
    list_xpath = "//a[contains(@href, 'Senators/s')]"

    def process_list_item(self, item):
        name = " ".join(item.xpath('.//text()'))
        name = re.sub(r'\s+', " ", name).replace(" ,", ",").strip()

        if 'Vacant' in name:
            return

        district = item.xpath("string(../../td[1])")
        party = item.xpath("string(../../td[2])")
        if party == 'Democrat':
            party = 'Democratic'

        leg_url = item.get('href')

        leg = Person(name=name, district=district, party=party, primary_org='upper', role='Senator')
        leg.add_link(leg_url)
        leg.add_source(self.url)
        leg.add_source(leg_url)

        self.get_page2(SenDetail, leg_url, obj=leg)

        return leg

class RepList(Page):
    url = "http://www.flhouse.gov/Sections/Representatives/representatives.aspx"
    list_xpath = '//div[@id="MemberListing"]/div[@class="rep_listing1"]'

    def process_list_item(self, item):
        link = item.xpath('.//div[@class="rep_style"]/a')[0]
        name = link.text_content().strip()

        if 'Vacant' in name or 'Resigned' in name or 'Pending' in name:
            return

        party = item.xpath('.//div[@class="party_style"]/text()')[0].strip()
        party = {'D': 'Democratic', 'R': 'Republican'}[party]

        district = item.xpath('.//div[@class="district_style"]/text()')[0].strip()

        leg_url = link.get('href')
        split_url = parse.urlsplit(leg_url)
        member_id = parse.parse_qs(split_url.query)['MemberId'][0]
        image = "http://www.flhouse.gov/FileStores/Web/Imaging/Member/{}.jpg".format(member_id)

        rep = Person(name=name, district=district, party=party, primary_org='lower',
                     role='Representative', image=image)
        rep.add_link(leg_url)
        rep.add_source(leg_url)
        rep.add_source(self.url)

        self.get_page2(RepDetail, leg_url, obj=rep)

        return rep

class RepDetail(Page):
    def process_page(self):
        self.scrape_office('Capitol Office')
        self.scrape_office('District Office')

    def scrape_office(self, name):
        pieces = [x.tail.strip() for x in
                  self.doc.xpath('//strong[text()="{}"]/following-sibling::br'.format(name))]

        if not pieces:
            # TODO: warn?
            return
        address = []

        for piece in pieces:
            if piece.startswith('Phone:'):
                # Phone: \r\n     (303) 111-2222
                if re.search(r'\d+', piece):
                    phone = piece.split(None, 1)[1]
                else:
                    phone = None
            else:
                address.append(re.sub(r'\s+', ' ', piece))

        type_ = 'capitol' if 'Capitol' in name else 'district'

        self.obj.add_contact_detail(type='address', value='\n'.join(address), note=type_)
        if phone:
            self.obj.add_contact_detail(type='voice', value=phone, note=type_)

class FlPersonScraper(Scraper, Spatula):

    def scrape(self):
        yield from self.get_page(SenList)
        yield from self.get_page(RepList)
<commit_msg>consume the iterable<commit_after>import re
from urllib import parse
from pupa.scrape import Scraper, Person
from .base import Spatula, Page

class SenDetail(Page):
    list_xpath = '//h4[contains(text(), "Office")]'

    def process_list_item(self, office):
        (name, ) = office.xpath('text()')
        if name == "Tallahassee Office":
            type_ = 'capitol'
        else:
            type_ = 'district'

        address_lines = [x.strip() for x in
                         office.xpath('following-sibling::div[1]')[0].text_content().splitlines()
                         if x.strip()]

        clean_address_lines = []
        fax = phone = None
        PHONE_RE = r'\(\d{3}\)\s\d{3}\-\d{4}'
        after_phone = False

        for line in address_lines:
            if re.search(r'(?i)open\s+\w+day', address_lines[0]):
                continue
            elif 'FAX' in line:
                fax = line.replace('FAX ', '')
                after_phone = True
            elif re.search(PHONE_RE, line):
                phone = line
                after_phone = True
            elif not after_phone:
                clean_address_lines.append(line)

        if phone:
            self.obj.add_contact_detail(type='voice', value=phone, note=type_)
        if fax:
            self.obj.add_contact_detail(type='fax', value=fax, note=type_)

        # address
        address = "\n".join(clean_address_lines)
        address = re.sub(r'\s{2,}', " ", address)
        if address:
            self.obj.add_contact_detail(type='address', value=address, note=type_)

    def process_page(self):
        list(super().process_page())
        email = self.doc.xpath('//a[contains(@href, "mailto:")]')[0].get('href').split(':')[-1]
        self.obj.add_contact_detail(type='email', value=email)

        self.obj.image = self.doc.xpath('//div[@id="sidebar"]//img/@src').pop()

class SenList(Page):
    url = "http://www.flsenate.gov/Senators/"
    list_xpath = "//a[contains(@href, 'Senators/s')]"

    def process_list_item(self, item):
        name = " ".join(item.xpath('.//text()'))
        name = re.sub(r'\s+', " ", name).replace(" ,", ",").strip()

        if 'Vacant' in name:
            return

        district = item.xpath("string(../../td[1])")
        party = item.xpath("string(../../td[2])")
        if party == 'Democrat':
            party = 'Democratic'

        leg_url = item.get('href')

        leg = Person(name=name, district=district, party=party, primary_org='upper', role='Senator')
        leg.add_link(leg_url)
        leg.add_source(self.url)
        leg.add_source(leg_url)

        self.get_page2(SenDetail, leg_url, obj=leg)

        return leg

class RepList(Page):
    url = "http://www.flhouse.gov/Sections/Representatives/representatives.aspx"
    list_xpath = '//div[@id="MemberListing"]/div[@class="rep_listing1"]'

    def process_list_item(self, item):
        link = item.xpath('.//div[@class="rep_style"]/a')[0]
        name = link.text_content().strip()

        if 'Vacant' in name or 'Resigned' in name or 'Pending' in name:
            return

        party = item.xpath('.//div[@class="party_style"]/text()')[0].strip()
        party = {'D': 'Democratic', 'R': 'Republican'}[party]

        district = item.xpath('.//div[@class="district_style"]/text()')[0].strip()

        leg_url = link.get('href')
        split_url = parse.urlsplit(leg_url)
        member_id = parse.parse_qs(split_url.query)['MemberId'][0]
        image = "http://www.flhouse.gov/FileStores/Web/Imaging/Member/{}.jpg".format(member_id)

        rep = Person(name=name, district=district, party=party, primary_org='lower',
                     role='Representative', image=image)
        rep.add_link(leg_url)
        rep.add_source(leg_url)
        rep.add_source(self.url)

        self.get_page2(RepDetail, leg_url, obj=rep)

        return rep

class RepDetail(Page):
    def process_page(self):
        self.scrape_office('Capitol Office')
        self.scrape_office('District Office')

    def scrape_office(self, name):
        pieces = [x.tail.strip() for x in
                  self.doc.xpath('//strong[text()="{}"]/following-sibling::br'.format(name))]

        if not pieces:
            # TODO: warn?
            return
        address = []

        for piece in pieces:
            if piece.startswith('Phone:'):
                # Phone: \r\n     (303) 111-2222
                if re.search(r'\d+', piece):
                    phone = piece.split(None, 1)[1]
                else:
                    phone = None
            else:
                address.append(re.sub(r'\s+', ' ', piece))

        type_ = 'capitol' if 'Capitol' in name else 'district'

        self.obj.add_contact_detail(type='address', value='\n'.join(address), note=type_)
        if phone:
            self.obj.add_contact_detail(type='voice', value=phone, note=type_)

class FlPersonScraper(Scraper, Spatula):

    def scrape(self):
        yield from self.get_page(SenList)
        yield from self.get_page(RepList)
<|endoftext|>


<commit_before>"""
Test SBprocess and SBThread APIs with printing of the stack traces using lldbutil.
"""

import os, time
import re
import unittest2
import lldb
from lldbtest import *

class ThreadsStackTracesTestCase(TestBase):

    mydir = TestBase.compute_mydir(__file__)

    def setUp(self):
        # Call super's setUp().
        TestBase.setUp(self)
        # Find the line number to break inside main().
        self.line = line_number('main.cpp', '// Set break point at this line.')

    # llvm.org/pr23043 - leaving the next two lines in so it's easy to find this
    # test will appear when searching for expectedFailure(Linux|i386)
    #@expectedFailureLinux
    #@expectedFailurei386
    @python_api_test
    def test_stack_traces(self):
        """Test SBprocess and SBThread APIs with printing of the stack traces."""
        self.buildDefault()
        self.break_and_print_stacktraces()

    def break_and_print_stacktraces(self):
        """Break at main.cpp:68 and do a threads dump"""
        if self.getArchitecture() in ['i386'] and "linux" in sys.platform:
            self.skipTest("Skipping because this test is known to fail on i386 Linux")

        exe = os.path.join(os.getcwd(), "a.out")

        target = self.dbg.CreateTarget(exe)
        self.assertTrue(target, VALID_TARGET)

        breakpoint = target.BreakpointCreateByLocation("main.cpp", self.line)
        self.assertTrue(breakpoint, VALID_BREAKPOINT)

        # Now launch the process, and do not stop at entry point.
        process = target.LaunchSimple (["abc", "xyz"], None, self.get_process_working_directory())

        if not process:
            self.fail("SBTarget.LaunchProcess() failed")

        import lldbutil
        if process.GetState() != lldb.eStateStopped:
            self.fail("Process should be in the 'stopped' state, "
                      "instead the actual state is: '%s'" %
                      lldbutil.state_type_to_str(process.GetState()))

        stacktraces = lldbutil.print_stacktraces(process, string_buffer=True)
        self.expect(stacktraces, exe=False,
            substrs = ['(int)argc=3'])

if __name__ == '__main__':
    import atexit
    lldb.SBDebugger.Initialize()
    atexit.register(lambda: lldb.SBDebugger.Terminate())
    unittest2.main()
<commit_msg>XFAILing TestPrintStackTraces on linux to stabilize the build bot<commit_after>"""
Test SBprocess and SBThread APIs with printing of the stack traces using lldbutil.
"""

import os, time
import re
import unittest2
import lldb
from lldbtest import *

class ThreadsStackTracesTestCase(TestBase):

    mydir = TestBase.compute_mydir(__file__)

    def setUp(self):
        # Call super's setUp().
        TestBase.setUp(self)
        # Find the line number to break inside main().
        self.line = line_number('main.cpp', '// Set break point at this line.')

    # fails on Linux i386 llvm.org/pr23043
    # fails 78/100 on Linux x86_64 when running against remote lldb-platform
    # failed 11/200 on the linux build bot
    @expectedFailureLinux
    @python_api_test
    def test_stack_traces(self):
        """Test SBprocess and SBThread APIs with printing of the stack traces."""
        self.buildDefault()
        self.break_and_print_stacktraces()

    def break_and_print_stacktraces(self):
        """Break at main.cpp:68 and do a threads dump"""

        exe = os.path.join(os.getcwd(), "a.out")

        target = self.dbg.CreateTarget(exe)
        self.assertTrue(target, VALID_TARGET)

        breakpoint = target.BreakpointCreateByLocation("main.cpp", self.line)
        self.assertTrue(breakpoint, VALID_BREAKPOINT)

        # Now launch the process, and do not stop at entry point.
        process = target.LaunchSimple (["abc", "xyz"], None, self.get_process_working_directory())

        if not process:
            self.fail("SBTarget.LaunchProcess() failed")

        import lldbutil
        if process.GetState() != lldb.eStateStopped:
            self.fail("Process should be in the 'stopped' state, "
                      "instead the actual state is: '%s'" %
                      lldbutil.state_type_to_str(process.GetState()))

        stacktraces = lldbutil.print_stacktraces(process, string_buffer=True)
        self.expect(stacktraces, exe=False,
            substrs = ['(int)argc=3'])

if __name__ == '__main__':
    import atexit
    lldb.SBDebugger.Initialize()
    atexit.register(lambda: lldb.SBDebugger.Terminate())
    unittest2.main()
<|endoftext|>


<commit_before>"""
KegPi

The tastiest solution to monitor your keggerator.
"""

import time

class FlowMeter(object):
    version = 'v0.1'
    enabled = True
    click_count = 0
    last_click_time = 0
    this_pour = 0
    last_pour = 0
    last_pour_time = 0
    total_pour = 0
    drink_count = 0
    time_now = 0
    calibration = 0
    ml_per_click = 3 #This parameter inherits from calibration method.
    oz_per_click = 0.08454
    ml_in_a_pint = 473.176
    ml_in_an_oz = 29.5735
    to_ml = 0
    last_pour_time = 0

    #Initialize all_the_things.jpg!!!
    def __init__(self):
        self.click_count = 0
        self.last_click_time = 0
        self.this_pour = 0
        self.last_pour = 0
        self.last_pour_time = 0
        self.total_pour = 0
        self.drink_count = 0
        self.enabled = True
        self.calibration = 0
        self.ml_per_click = self.calibration
        self.oz_per_click = 0.08454
        self.ml_in_a_pint = 473.176
        self.ml_in_an_oz = 29.5735
        self.to_ml = 0
        self.last_pour_time = 0

    #User will input the calibration
    #Need to add logic to detect if user doesn't eneter proper calibration.
    def calibrate(self):
        if (self.calibration > 0):
            print "Calibrated", self.calibration
        else:
            print "Please measure your last pour in ml, and then enter the volume:"
            print "Hint: The bigger the pour, the more accurate the calibration."
            cal_input = raw_input("> ")
            if cal_input == '':
                print "You didn't input anything, please try again."
                cal_input = 0
            elif cal_input.isdigit() != True: #This doesn't seem to work for floats.
                print "That is not a number. Please retry the calibration."
                cal_input = 0
            else:
                self.calibration = (float(cal_input) / self.last_pour)
                print self.calibration, "ml in a click."

    #GPIO detects the rising edge, and updates the current count.
    #Find the time of the last click.
    def update(self):
        self.click_count = self.click_count + 1
        self.last_click_time = time.time()
        print self.click_count, "Last Click"
        print self.last_click_time, "was the time this cick was found."

    #If no clicks are found in the last 20 seconds, record the clicks as a last pour.
    #Reset click count for next pour.
    def last_pour_func(self):
        if (self.click_count == 0):
            time.sleep(1)
            print "Nothing to do yet."
            self.last_pour_func()
        elif (time.time() - self.last_click_time > 5):
            self.last_pour = self.click_count
            self.click_count = 0
            print "Last pour was", self.last_pour, " clicks."
            print "Click count reset to: ", self.click_count
            self.last_pour_time = time.time()
            if (self.calibration == 0):
                self.calibrate()
            else:
                print "Do nothing"
            #Call this to update our volumes and totals in one go.
            self.update_all()
            return self.last_pour
        else:
            print "Nothing happened."
            time.sleep(1)
            self.last_pour_func()

    #Update all of these in one call.
    def update_all(self):
        self.last_pour_in_ml()
        self.last_pour_in_oz()
        self.store_total_clicks()
        self.count_drinks()

    #Store the total click count to update keg volume.
    def store_total_clicks(self):
        self.total_pour = self.last_pour + self.total_pour
        return self.total_pour

    #We can use this to determine how many beers total were had.
    def count_drinks(self):
        self.drink_count = self.drink_count =+ 1
        print self.drink_count, "drinks total."
        return self.drink_count

    #Store the last pour in ml
    def last_pour_in_ml(self):
        if self.calibration == 0:
            print "Please calibrate by calling calibrate() method first."
        else:
            self.to_ml = (self.last_pour * self.calibration)
            print self.to_ml, "ml poured."
            return self.to_ml

    #Store the last pour in OZ
    def last_pour_in_oz(self):
        if self.calibration == 0:
            print "Please calibrate by calling calibrate() method first."
        else:
            self.last_pour_oz = ((self.last_pour * self.calibration) / self.ml_in_an_oz)
            print self.last_pour_oz, "oz poured."
            return self.last_pour_oz

<commit_msg>Click order of operations<commit_after>"""
KegPi

The tastiest solution to monitor your keggerator.
"""

import time

class FlowMeter(object):
    version = 'v0.1'
    enabled = True
    click_count = 0
    last_click_time = 0
    this_pour = 0
    last_pour = 0
    last_pour_time = 0
    total_pour = 0
    drink_count = 0
    time_now = 0
    calibration = 0
    ml_per_click = 3 #This parameter inherits from calibration method.
    oz_per_click = 0.08454
    ml_in_a_pint = 473.176
    ml_in_an_oz = 29.5735
    to_ml = 0
    last_pour_time = 0

    #Initialize all_the_things.jpg!!!
    def __init__(self):
        self.click_count = 0
        self.last_click_time = 0
        self.this_pour = 0
        self.last_pour = 0
        self.last_pour_time = 0
        self.total_pour = 0
        self.drink_count = 0
        self.enabled = True
        self.calibration = 0
        self.ml_per_click = self.calibration
        self.oz_per_click = 0.08454
        self.ml_in_a_pint = 473.176
        self.ml_in_an_oz = 29.5735
        self.to_ml = 0
        self.last_pour_time = 0

    #User will input the calibration
    #Need to add logic to detect if user doesn't eneter proper calibration.
    def calibrate(self):
        if (self.calibration > 0):
            print "Calibrated", self.calibration
        else:
            print "Please measure your last pour in ml, and then enter the volume:"
            print "Hint: The bigger the pour, the more accurate the calibration."
            cal_input = raw_input("> ")
            if cal_input == '':
                print "You didn't input anything, please try again."
                cal_input = 0
            elif cal_input.isdigit() != True: #This doesn't seem to work for floats.
                print "That is not a number. Please retry the calibration."
                cal_input = 0
            else:
                self.calibration = (float(cal_input) / self.last_pour)
                print self.calibration, "ml in a click."

    #GPIO detects the rising edge, and updates the current count.
    #Find the time of the last click.
    def update(self):
        self.click_count = self.click_count + 1
        self.last_click_time = time.time()
        print self.click_count, "Last Click"
        print self.last_click_time, "was the time this cick was found."

    #If no clicks are found in the last 20 seconds, record the clicks as a last pour.
    #Reset click count for next pour.
    def last_pour_func(self):
        if (self.click_count == 0):
            time.sleep(1)
            print "Nothing to do yet."
            self.last_pour_func()
        elif (time.time() - self.last_click_time > 5):
            self.last_pour = self.click_count
            self.click_count = 0
            print "Click count reset to: ", self.click_count
            self.last_pour_time = time.ctime()
            if (self.calibration == 0):
                self.calibrate()
            else:
                print "Do nothing."
            #Call this to update our volumes and totals in one go.
            self.update_all()
            print "Last pour was", self.last_pour, "clicks, at", self.last_pour_time
            return self.last_pour
        else:
            print "Waiting for pour to finish."
            time.sleep(1)
            self.last_pour_func()

    #Update all of these in one call.
    def update_all(self):
        self.last_pour_in_ml()
        self.last_pour_in_oz()
        self.store_total_clicks()
        self.count_drinks()

    #Store the total click count to update keg volume.
    def store_total_clicks(self):
        self.total_pour = self.last_pour + self.total_pour
        return self.total_pour

    #We can use this to determine how many beers total were had.
    def count_drinks(self):
        self.drink_count = self.drink_count =+ 1
        print self.drink_count, "drinks total."
        return self.drink_count

    #Store the last pour in ml
    def last_pour_in_ml(self):
        if self.calibration == 0:
            print "Please calibrate by calling calibrate() method first."
        else:
            self.to_ml = (self.last_pour * self.calibration)
            print self.to_ml, "ml poured."
            return self.to_ml

    #Store the last pour in OZ
    def last_pour_in_oz(self):
        if self.calibration == 0:
            print "Please calibrate by calling calibrate() method first."
        else:
            self.last_pour_oz = ((self.last_pour * self.calibration) / self.ml_in_an_oz)
            print self.last_pour_oz, "oz poured."
            return self.last_pour_oz

<|endoftext|>


<commit_before>"""Ipython parallel ready entry points for parallel execution
"""
import contextlib

from IPython.parallel import require

from bcbio.pipeline import sample, lane, shared, variation
from bcbio.variation import bamprep, realign, genotype, ensemble, recalibrate, multi
from bcbio.log import setup_logging, logger

@contextlib.contextmanager
def _setup_logging(args):
    if len(args) > 0:
        for check_i in [0, -1]:
            config = args[0][check_i]
            if isinstance(config, dict) and config.has_key("config"):
                config = config["config"]
                break
            elif isinstance(config, dict) and config.has_key("algorithm"):
                break
            else:
                config = None
        setup_logging(config)
    try:
        yield None
    except:
        logger.exception("Unexpected error")
        raise

@require(lane)
def process_lane(*args):
    with _setup_logging(args):
        return apply(lane.process_lane, *args)

@require(lane)
def trim_lane(*args):
    with _setup_logging(args):
        return apply(lane.trim_lane, *args)

@require(lane)
def process_alignment(*args):
    with _setup_logging(args):
        return apply(lane.process_alignment, *args)
process_alignment.metadata = {"resources": ["novoalign", "bwa"]}

@require(lane)
def align_prep_full(*args):
    with _setup_logging(args):
        return apply(lane.align_prep_full, *args)
process_alignment.metadata = {"resources": ["novoalign", "bwa", "gatk"]}

@require(sample)
def merge_sample(*args):
    with _setup_logging(args):
        return apply(sample.merge_sample, *args)

@require(sample)
def recalibrate_sample(*args):
    with _setup_logging(args):
        return apply(sample.recalibrate_sample, *args)

@require(recalibrate)
def prep_recal(*args):
    with _setup_logging(args):
        return apply(recalibrate.prep_recal, *args)
prep_recal.metadata = {"resources": ["gatk"]}

@require(recalibrate)
def write_recal_bam(*args):
    with _setup_logging(args):
        return apply(recalibrate.write_recal_bam, *args)

@require(realign)
def realign_sample(*args):
    with _setup_logging(args):
        return apply(realign.realign_sample, *args)

@require(multi)
def split_variants_by_sample(*args):
    with _setup_logging(args):
        return apply(multi.split_variants_by_sample, *args)

@require(bamprep)
def piped_bamprep(*args):
    with _setup_logging(args):
        return apply(bamprep.piped_bamprep, *args)

@require(sample)
def postprocess_variants(*args):
    with _setup_logging(args):
        return apply(sample.postprocess_variants, *args)

@require(sample)
def process_sample(*args):
    with _setup_logging(args):
        return apply(sample.process_sample, *args)

@require(sample)
def generate_transcript_counts(*args):
    with _setup_logging(args):
        return apply(sample.generate_transcript_counts, *args)

@require(sample)
def generate_bigwig(*args):
    with _setup_logging(args):
        return apply(sample.generate_bigwig, *args)

@require(shared)
def combine_bam(*args):
    with _setup_logging(args):
        return apply(shared.combine_bam, *args)

@require(genotype)
def variantcall_sample(*args):
    with _setup_logging(args):
        return apply(genotype.variantcall_sample, *args)

@require(genotype)
def combine_variant_files(*args):
    with _setup_logging(args):
        return apply(genotype.combine_variant_files, *args)

@require(variation)
def detect_sv(*args):
    with _setup_logging(args):
        return apply(variation.detect_sv, *args)

@require(ensemble)
def combine_calls(*args):
    with _setup_logging(args):
        return apply(ensemble.combine_calls, *args)
<commit_msg>Correctly set ipython metadata for piped alignment preparation<commit_after>"""Ipython parallel ready entry points for parallel execution
"""
import contextlib

from IPython.parallel import require

from bcbio.pipeline import sample, lane, shared, variation
from bcbio.variation import bamprep, realign, genotype, ensemble, recalibrate, multi
from bcbio.log import setup_logging, logger

@contextlib.contextmanager
def _setup_logging(args):
    if len(args) > 0:
        for check_i in [0, -1]:
            config = args[0][check_i]
            if isinstance(config, dict) and config.has_key("config"):
                config = config["config"]
                break
            elif isinstance(config, dict) and config.has_key("algorithm"):
                break
            else:
                config = None
        setup_logging(config)
    try:
        yield None
    except:
        logger.exception("Unexpected error")
        raise

@require(lane)
def process_lane(*args):
    with _setup_logging(args):
        return apply(lane.process_lane, *args)

@require(lane)
def trim_lane(*args):
    with _setup_logging(args):
        return apply(lane.trim_lane, *args)

@require(lane)
def process_alignment(*args):
    with _setup_logging(args):
        return apply(lane.process_alignment, *args)
process_alignment.metadata = {"resources": ["novoalign", "bwa"]}

@require(lane)
def align_prep_full(*args):
    with _setup_logging(args):
        return apply(lane.align_prep_full, *args)
align_prep_full.metadata = {"resources": ["novoalign", "bwa", "gatk"]}

@require(sample)
def merge_sample(*args):
    with _setup_logging(args):
        return apply(sample.merge_sample, *args)

@require(sample)
def recalibrate_sample(*args):
    with _setup_logging(args):
        return apply(sample.recalibrate_sample, *args)

@require(recalibrate)
def prep_recal(*args):
    with _setup_logging(args):
        return apply(recalibrate.prep_recal, *args)
prep_recal.metadata = {"resources": ["gatk"]}

@require(recalibrate)
def write_recal_bam(*args):
    with _setup_logging(args):
        return apply(recalibrate.write_recal_bam, *args)

@require(realign)
def realign_sample(*args):
    with _setup_logging(args):
        return apply(realign.realign_sample, *args)

@require(multi)
def split_variants_by_sample(*args):
    with _setup_logging(args):
        return apply(multi.split_variants_by_sample, *args)

@require(bamprep)
def piped_bamprep(*args):
    with _setup_logging(args):
        return apply(bamprep.piped_bamprep, *args)

@require(sample)
def postprocess_variants(*args):
    with _setup_logging(args):
        return apply(sample.postprocess_variants, *args)

@require(sample)
def process_sample(*args):
    with _setup_logging(args):
        return apply(sample.process_sample, *args)

@require(sample)
def generate_transcript_counts(*args):
    with _setup_logging(args):
        return apply(sample.generate_transcript_counts, *args)

@require(sample)
def generate_bigwig(*args):
    with _setup_logging(args):
        return apply(sample.generate_bigwig, *args)

@require(shared)
def combine_bam(*args):
    with _setup_logging(args):
        return apply(shared.combine_bam, *args)

@require(genotype)
def variantcall_sample(*args):
    with _setup_logging(args):
        return apply(genotype.variantcall_sample, *args)

@require(genotype)
def combine_variant_files(*args):
    with _setup_logging(args):
        return apply(genotype.combine_variant_files, *args)

@require(variation)
def detect_sv(*args):
    with _setup_logging(args):
        return apply(variation.detect_sv, *args)

@require(ensemble)
def combine_calls(*args):
    with _setup_logging(args):
        return apply(ensemble.combine_calls, *args)
<|endoftext|>


<commit_before>import uuid
import gym

class EnvironmentManager(object):
    """
    Environment Manager

    Control the environments that will be handled by the server.
    """
    def __init__(self):
        self.envs = {}
        self.id_len = 8

    def create(self, env_id):
        """
        Create a new environment.

        :param env_id: Valid gym environment.
        :return: Instance ID
        """
        env = gym.make(env_id)
        instance_id = str(uuid.uuid4().hex)[:self.id_len]
        self.envs[instance_id] = env
        return instance_id

    def reset(self, instance_id):
        """
        Reset state of a given instance id.
        :param instance_id:
        :return: Array of observations.
        """
        env = self.envs[instance_id]
        obs = env.reset()
        return env.observation_space.to_jsonable(obs)

    def step(self, instance_id, action):
        """
        Execute action given by agent on the given environment
        :param instance_id: instance id of the environment
        :param action: action
        :return: Dictionary with observation, reward, done flag and info.
        """
        env = self.envs[instance_id]

        action_from_json = int(env.action_space.from_jsonable(action))

        try:
            observation, reward, done, info = env.step(action_from_json)

            environment_response = {
                'observation': env.observation_space.to_jsonable(observation),
                'reward': reward,
                'done': done,
                'info': info
            }

            return environment_response
        except gym.error.ResetNeeded as ex:
            return {'error': ex.message}

    def monitor_start(self, instance_id, force, resume):
        """
        Start Monitoring.

        The directory will be the same as the instace_id.

        The video will be available on the port server:8000/instance_id
        :param instance_id: instance id of the desired environment.
        :param force: Clear out existing training data from this directory (by deleting every file prefixed with "openaigym.").
        :param resume: Retain the training data already in this directory, which will be merged with our new data.
        :return:
        """
        env = self.envs[instance_id]

        directory = "monitor/{}".format(instance_id)

        env.monitor.start(directory, force=force, resume=resume)

    def monitor_close(self, instance_id):
        """
        Stop Monitor
        :param instance_id: instance id.
        :return: None.
        """
        env = self.envs[instance_id]
        env.monitor.close()

    def render(self, instance_id):
        env = self.envs[self.envs]

        render_mode = get_render_mode(env)

def get_render_mode(environ):
    if 'rgb_array' in environ.metadata['render.modes']:
        return 'rgb_array'
    elif 'ansi' in environ.metadata['render.modes']:
        return 'ansi'
    else:
        return None

def save_img(img):
    im = Image.fromarray(img)
    im.save("simulation.png")
<commit_msg>Add docstrings and more functionality<commit_after>import uuid
import gym

class EnvironmentManager(object):
    """
    Environment Manager

    Control the environments that will be handled by the server.
    """

    TRAINING_DIRECTORY = "monitor/{}"

    def __init__(self):
        self.envs = {}
        self.id_len = 8

    def create(self, env_id):
        """
        Create a new environment.

        :param env_id: Valid gym environment.
        :return: Instance ID
        """
        env = gym.make(env_id)
        instance_id = str(uuid.uuid4().hex)[:self.id_len]
        self.envs[instance_id] = env
        return instance_id

    def reset(self, instance_id):
        """
        Reset state of a given instance id.
        :param instance_id:
        :return: Array of observations.
        """
        env = self.envs[instance_id]
        obs = env.reset()
        return env.observation_space.to_jsonable(obs)

    def step(self, instance_id, action, render=False):
        """
        Execute action given by agent on the given environment
        :param instance_id: instance id of the environment
        :param action: action
        :return: Dictionary with observation, reward, done flag and info.
        """
        env = self.envs[instance_id]

        action_from_json = int(env.action_space.from_jsonable(action))

        observation, reward, done, info = env.step(action_from_json)

        environment_response = {
            'observation': env.observation_space.to_jsonable(observation),
            'reward': reward,
            'done': done,
            'info': info
        }

        return environment_response

    def monitor_start(self, instance_id, force, resume):
        """
        Start Monitoring.

        The directory will be the same as the instace_id.

        The video will be available on the port server:8000/instance_id
        :param instance_id: instance id of the desired environment.
        :param force: Clear out existing training data from this directory (by deleting every file prefixed with "openaigym.").
        :param resume: Retain the training data already in this directory, which will be merged with our new data.
        :return:
        """
        env = self.envs[instance_id]

        directory = EnvironmentManager.TRAINING_DIRECTORY.format(instance_id)

        env.monitor.start(directory, force=force, resume=resume)

    def monitor_close(self, instance_id):
        """
        Stop Monitor
        :param instance_id: instance id.
        :return: None.
        """
        env = self.envs[instance_id]
        env.monitor.close()

    def upload(self, instance_id, algorithm_id, writeup, api_key, ignore_open_monitors):
        """
        Upload training information created with monitor.
        :param instance_id: Id of the environment that was trained.
        :param algorithm_id: An arbitrary string indicating the paricular version of the algorithm
               (including choices of parameters) you are running.
        :param writeup: A Gist URL (of the form https://gist.github.com/<user>/<id>)
                        containing your writeup for this evaluation.
        :param api_key:  Your OpenAI API key. Can also be provided as an environment variable (OPENAI_GYM_API_KEY).
        :param ignore_open_monitors: Ignore open monitors when uploading.
        :return:
        """
        directory = EnvironmentManager.TRAINING_DIRECTORY.format(instance_id)

        gym.upload(directory, algorithm_id, writeup, api_key,
                   ignore_open_monitors)

    def info(self, instance_id):
        """
        Expose useful information, such as: action_space, and observation_space.
        :param instance_id:
        :return:
        """
        env = self.envs[instance_id]
        return {
            'action_space': str(env.action_space),
            'observation_space': {
                'shape': env.observation_space.shape,
                'low': env.observation_space.low.tolist(),
                'high': env.observation_space.high.tolist()
            }
        }

    def render(self, instance_id):
        raise NotImplementedError()

<|endoftext|>


<commit_before>#!/usr/bin/env python

import rospy
import math
from geometry_msgs.msg import Point
from geometry_msgs.msg import Twist, Vector3

# Unit of distance is meter, angle is degree
DISTANCE_UPPER_LIMIT = 1.0
DISTANCE_LOWER_LIMIT = 0.5
ANGLE_UPPER_LIMIT = 10
ANGLE_LOWER_LIMIT = -10

def callback_people(People_Position):
    global Linear_Vel
    global Angular_Vel

    distance_dep = People_Position.z/1000
    distance_hor = People_Position.x/1000
    if distance_dep != 0:
        angular = (math.atan(distance_hor/distance_dep)*360)/(2*math.pi)
    else:
		angular = 0
    # linear velocity controller
    if distance_dep > DISTANCE_UPPER_LIMIT:
        Linear_Vel = 0.2 + 0.3*(distance_dep - DISTANCE_UPPER_LIMIT)/distance_dep
    elif distance_dep < DISTANCE_LOWER_LIMIT and distance_dep != 0:
        Linear_Vel = -0.2
    else:
        Linear_Vel = 0

    # angular velocity controller
    if angular > ANGLE_UPPER_LIMIT:
        Angular_Vel = 0.2 + 0.2*(angular - ANGLE_UPPER_LIMIT)/angular
    elif angular < ANGLE_LOWER_LIMIT:
        Angular_Vel = -0.2 - 0.2*(angular - ANGLE_UPPER_LIMI)/angular
    else:
        Angular_Vel = 0

def main():
    global Linear_Vel
    global Angular_Vel
    rospy.init_node('Follow_Node')
    rospy.Subscriber('/nearest/position', Point, callback=callback_people, queue_size=10)
    vel_pub = rospy.Publisher('/cmd_vel_mux/input/navi', Twist, queue_size=1)

    rate = rospy.Rate(3)
    while True:
        #print 'hello, world'
        vel_msg = Twist(Vector3(Linear_Vel,0,0), Vector3(0,0,Angular_Vel))
        vel_pub.publish(vel_msg)
        print ("velocity:(%s,0,0) (0,0,%s)"%(Linear_Vel,Angular_Vel))
        rate.sleep()

if __name__=='__main__':
    main()
<commit_msg>Velocity smoother added...<commit_after>#!/usr/bin/env python

import rospy
import math
from geometry_msgs.msg import Point
from geometry_msgs.msg import Twist, Vector3

# Unit of distance is meter, angle is degree
DISTANCE_UPPER_LIMIT = 1.0
DISTANCE_LOWER_LIMIT = 0.5
ANGLE_UPPER_LIMIT = 10
ANGLE_LOWER_LIMIT = -10

def callback_people(People_Position):
    global Linear_Vel
    global Angular_Vel

    distance_dep = People_Position.z/1000
    distance_hor = People_Position.x/1000
    if distance_dep != 0:
        angular = (math.atan(distance_hor/distance_dep)*360)/(2*math.pi)
    else:
		angular = 0
    # linear velocity controller
    if distance_dep > DISTANCE_UPPER_LIMIT:
        Linear_Vel = 0.1 + 0.3*(distance_dep - DISTANCE_UPPER_LIMIT)
    elif distance_dep < DISTANCE_LOWER_LIMIT and distance_dep != 0:
        Linear_Vel = -0.1
    else:
        Linear_Vel = 0

    # angular velocity controller
    if angular > ANGLE_UPPER_LIMIT:
        Angular_Vel = 0.2 + 0.4*(angular - ANGLE_UPPER_LIMIT)/angular
    elif angular < ANGLE_LOWER_LIMIT:
        Angular_Vel = -0.2 - 0.4*(angular - ANGLE_LOWER_LIMIT)/angular
    else:
        Angular_Vel = 0

def main():
    global Linear_Vel
    global Angular_Vel
    Linear_Vel = 0
    Angular_Vel = 0

    rospy.init_node('Follow_Node')
    rospy.Subscriber('/nearest/position', Point, callback=callback_people, queue_size=10)
    vel_pub = rospy.Publisher('/cmd_vel_mux/input/navi', Twist, queue_size=1)

    rate = rospy.Rate(3)
    while not rospy.is_shutdown():
        vel_msg = Twist(Vector3(Linear_Vel,0,0), Vector3(0,0,Angular_Vel))
        vel_pub.publish(vel_msg)
        print ("velocity:(%s,0,0) (0,0,%s)"%(Linear_Vel,Angular_Vel))
        rate.sleep()

if __name__=='__main__':
    main()
<|endoftext|>


<commit_before>"""This file contains code for use with "Think Bayes",
by Allen B. Downey, available from greenteapress.com

Copyright 2014 Allen B. Downey
License: GNU GPLv3 http://www.gnu.org/licenses/gpl.html
"""

from __future__ import print_function, division

import numpy
import thinkbayes2
import thinkplot

class Football(thinkbayes2.Suite):
    """Represents hypotheses about."""

    def Likelihood(self, data, hypo):
        """Computes the likelihood of the data under the hypothesis.

        hypo: hypothetical goal scoring rate in goals per game
        data: time between goals in minutes

        x=quantity where evaluating print_function
        lam=parameter determining rate in events/unit time
        """
        x = data #time between goals in minutes
        lam = hypo/60 #time per minute: the 11 in the update is 11 minutes into the game
        like = thinkbayes2.EvalExponentialPdf(x,lam) #evaluating for every value of lamda
        return like

    def PredRemaining(self, rem_time, score):
        """Plots the predictive distribution for final number of goals.

        rem_time: remaining time in the game in minutes
        score: number of goals already scored
        """
        metapmf=thinkbayes2.Pmf() #PMF about PMFS. probabilities of pmf values
        for lam, prob in self.Items(): #loop through probabilities of lamdas
            #print(lam,prob)
            lt = lam*rem_time/60
            pmf=thinkbayes2.MakePoissonPmf(lt,20)
            #thinkplot.Pdf(pmf,linewidth=1,alpha=0.2,color='purple')
            metapmf[pmf]=prob

        mix = thinkbayes2.MakeMixture(metapmf)
        mix += score #shift by 2 because we've already seen 2
        return mix

def main():
    meanFG=1.647
    meanTD=2.397
    FGtime=60/meanFG
    TDtime=60/meanTD

    hyposFG = numpy.linspace(0, 20, 201)
    suiteFG = Football(hyposFG)

    thinkplot.Pdf(suiteFG, label='priorFG')
    print('Field Goal prior mean', suiteFG.Mean())

    hyposTD = numpy.linspace(0, 20, 201)
    suiteTD = Football(hyposTD)

    thinkplot.Pdf(suiteFG, label='priorTD')
    print('Touchdown prior mean', suiteTD.Mean())

    #construct a prior using pseudo-observation
    suiteFG.Update(FGtime)
    thinkplot.Pdf(suiteFG, label='prior 2: FG pseudo-observation')
    print('pseudo-observation', suiteFG.Mean())

    #construct a prior using pseudo-observation
    suiteTD.Update(TDtime)
    thinkplot.Pdf(suiteTD, label='prior 2: TD pseudo-observation')
    print('pseudo-observation', suiteTD.Mean())

    #update with real data
    #suite.Update(11)
    #thinkplot.Pdf(suite, label='posterior 1 after pseudo-observation')
    #print('after one goal', suite.Mean())

    #revised belief that goal scoring rate (x axis) is higher

    #suite.Update(12)
    #thinkplot.Pdf(suite, label='posterior 2')
    #print('after two goals', suite.Mean())
    thinkplot.Show()

    #if we know what lamda is, we know the goals left in the game.
    FGpredict=suiteFG.PredRemaining(60,0)
    TDpredict=suiteTD.PredRemaining(60,0)
    GoalTotal=FGpredict*3+TDpredict*7
    thinkplot.Clf()
    #thinkplot.Hist(FGpredict)
    #thinkplot.Hist(TDpredict)
    thinkplot.Hist(GoalTotal)
    thinkplot.Show()

if __name__ == '__main__':
    main()
<commit_msg>Refactored priors into a function<commit_after>"""This file contains code for use with "Think Bayes",
by Allen B. Downey, available from greenteapress.com

Copyright 2014 Allen B. Downey
License: GNU GPLv3 http://www.gnu.org/licenses/gpl.html
"""

from __future__ import print_function, division

import numpy
import thinkbayes2
import thinkplot

class Football(thinkbayes2.Suite):
    """Represents hypotheses about."""

    def Likelihood(self, data, hypo):
        """Computes the likelihood of the data under the hypothesis.

        hypo: hypothetical goal scoring rate in goals per game
        data: time between goals in minutes

        x=quantity where evaluating print_function
        lam=parameter determining rate in events/unit time
        """
        x = data #time between goals in minutes
        lam = hypo/60 #time per minute: the 11 in the update is 11 minutes into the game
        like = thinkbayes2.EvalExponentialPdf(x,lam) #evaluating for every value of lamda
        return like

    def PredRemaining(self, rem_time, score):
        """Plots the predictive distribution for final number of goals.

        rem_time: remaining time in the game in minutes
        score: number of goals already scored
        """
        metapmf=thinkbayes2.Pmf() #PMF about PMFS. probabilities of pmf values
        for lam, prob in self.Items(): #loop through probabilities of lamdas
            #print(lam,prob)
            lt = lam*rem_time/60
            pmf=thinkbayes2.MakePoissonPmf(lt,20)
            #thinkplot.Pdf(pmf,linewidth=1,alpha=0.2,color='purple')
            metapmf[pmf]=prob

        mix = thinkbayes2.MakeMixture(metapmf)
        mix += score #shift by 2 because we've already seen 2
        return mix

def constructPriors():
    meanFG=1.647
    meanTD=2.397
    FGtime=60/meanFG
    TDtime=60/meanTD

    hyposFG = numpy.linspace(0, 20, 201)
    suiteFG = Football(hyposFG)

    hyposTD = numpy.linspace(0, 20, 201)
    suiteTD = Football(hyposTD)

    thinkplot.Pdf(suiteFG, label='priorFG')
    print('Field Goal prior mean', suiteFG.Mean())
    thinkplot.Pdf(suiteFG, label='priorTD')
    print('Touchdown prior mean', suiteTD.Mean())

    ##construct priors using pseudo-observation
    suiteFG.Update(FGtime)
    suiteTD.Update(TDtime)

    thinkplot.Pdf(suiteFG, label='prior 2: FG pseudo-observation')
    print('pseudo-observation', suiteFG.Mean())
    thinkplot.Pdf(suiteTD, label='prior 2: TD pseudo-observation')
    print('pseudo-observation', suiteTD.Mean())

    thinkplot.Show()

    return suiteTD, suiteFG

def main():

    suiteTD, suiteFG = constructPriors()

    #if we know what lamda is, we know the goals left in the game.
    FGpredict=suiteFG.PredRemaining(60,0)
    TDpredict=suiteTD.PredRemaining(60,0)
    GoalTotal=FGpredict*3+TDpredict*7
    thinkplot.Clf()
    #thinkplot.Hist(FGpredict)
    #thinkplot.Hist(TDpredict)
    thinkplot.Hist(GoalTotal)
    thinkplot.Show()

if __name__ == '__main__':
    main()
<|endoftext|>