<commit_before># -*- coding: utf-8 -*-
import numpy as np
from scipy.ndimage.measurements import maximum_position, label, find_objects
from scipy.ndimage.morphology import generate_binary_structure
from skimage.measure import regionprops
from skimage.morphology import dilation
import matplotlib.pyplot as plt
from candidates import Candidate
from astropy.time import TimeDelta

def find_levels_bcp(ts, prob=0.95, p0=0.0001, burnin=500, mcmc=1000, w0=0.2):
    kwargs = {'p0': p0, 'burnin': burnin, 'mcmc': mcmc, 'w0': w0}
    ts = FloatVector(ts)
    # Load library ``bcp``
    bcp = importr("bcp")
    out = bcp.bcp(ts, **kwargs)
    posterior_probs = out.rx2("posterior.prob")
    return np.where(posterior_probs > prob)[0]

def find_levels_regtree(ts):
    """
    Attempt to find regions of high / low signal using decision tree regression.

    If RFI is time-limited then dynamical spectra will show `stripes` of
    enhanced noise. This function will find such stripes.

    :param ts:
        Iterable of signal values.
    :return:
    """
    import scipy
    from sklearn.tree import DecisionTreeRegressor
    from sklearn.grid_search import GridSearchCV
    x = np.arange(len(ts)).reshape((len(ts), 1))

    def get_score(depth, rnd=42):
        max_depth = depth
        x = np.arange(len(ts)).reshape((len(ts), 1))
        # ts = scipy.signal.medfilt(ts, 101)
        # param_grid = {'learning_rate': [0.3, 0.1, 0.05, 0.01],
        #               'max_depth': [3, 4, 6],
        #               'min_samples_leaf': [2, 3, 9, 17],
        #               'max_features': [1.0, 0.3, 0.1]}
        # est = GradientBoostingClassifier(n_estimators=3000)
        # gs_cv = GridSearchCV(est, param_grid, n_jobs=4).fit(X_scaled, y)
        # gs_cv.best_params_
        # est = GradientBoostingClassifier(n_estimators=3000, **gs_cv.best_params_)
        # est.fit(X_scaled, y)
        # param_grid = {'max_depth': [2 * i for i in range(1, 100, 10)],
        #               'min_samples_split': [2, 20, 100, 1000]}
        # param_grid = {'min_samples_split': [2, 20, 100, 1000],
        #               'min_samples_leaf': [5, 50, 100, 1000]}
        regr = DecisionTreeRegressor(max_depth=max_depth, min_samples_split=1000,
                                     min_samples_leaf=1000, random_state=rnd,
                                     max_leaf_nodes=max_depth)
        # from sklearn.cross_validation import KFold
        # kf = KFold(len(ts), n_folds=10, shuffle=True)
        # gs_cv = GridSearchCV(regr, param_grid, n_jobs=4, cv=kf).fit(x, ts)
        # gs_cv.best_params_
        # regr = DecisionTreeRegressor(max_depth=max_depth, **gs_cv.best_params_)
        regr.fit(x, ts)
        return regr.score(x, ts)

    def plot_tree(max_depth, rnd=42):
        regr = DecisionTreeRegressor(max_depth=max_depth, min_samples_split=1000,
                                     min_samples_leaf=1000, random_state=rnd,
                                     max_leaf_nodes=max_depth)
        regr.fit(x, ts)
        x_test = np.arange(len(ts)).reshape((len(ts), 1))
        y_test = regr.predict(x_test)
        plt.plot(ts, '.k')
        plt.plot(x_test, y_test)
    # x1 = regr.tree_.threshold[0]
    # y_before, y_after = np.squeeze(regr.tree_.value)[1:]

    scores = np.array([get_score(depth) for depth in range(2, 100)])
    # TODO: Find break that corresponds to big differences.
    plt.plot(range(2, 100), scores)

def find_peaks(array_like, n_std=4, med_width=31, gauss_width=2):
    """
    Find peaks in 1D array.

    Data first median filtered with window size ``med_width``. Then it convolved
    with gaussian filter of width ``gauss_width``. Indexes of data with values
    higher then ``n_std`` are returned.

    :param array_like:
        Iterable of data values.
    :param med_width: (optional)
        Width of median filter to preprocess data. (default: ``30``)
    :param gauss_width: (optional)
        Width of gaussian filter to convolve median filtered data. (default:
        ``2``)
    :param n_std: (optional)
        Number of standard deviations to consider when searching peaks.
        (default: 4)

    :return:
        Numpy array of indexes of peak values.
    """
    import scipy
    array = np.asarray(array_like)
    array = scipy.signal.medfilt(array, med_width)
    garray = scipy.ndimage.filters.gaussian_filter1d(array, gauss_width)
    ind = (garray - np.mean(garray)) > n_std * np.std(garray)
    return ind

def find_noisy(dsp, n, axis=0, rnd=42):
    """
    Attempt to find noisy parts of dynamical spectra. Fitting gaussian mixture
    model to histogram of ``dsp`` value. If 2 components is superior model based
    on BIC then we can find typical threshold for high-amplitude noise.
    :param dsp:
    :param n:
    :param axis:
    :return:
    """
    from sklearn.mixture import GMM
    results = dict()
    data = dsp.reshape((dsp.size, 1))
    for i in range(1, n + 1):
        classif = GMM(n_components=i)
        classif.fit(data)
        results.update({classif.bic(data): [i, classif]})
    min_bic = min(results.keys())
    i, clf = results[min_bic]
    if i == 2:
        threshold = (clf.means_ + 3. * np.sqrt(clf.covars_))[0][0]
        ts = np.mean(dsp, axis=0)
        import scipy
        tsf = scipy.signal.medfilt(ts, 3)
        dsp_changes = tsf < threshold
        n_peaks = len(np.where(dsp_changes[:-1] != dsp_changes[1:])[0])
        regr = DecisionTreeRegressor(max_depth=n_peaks, min_samples_split=50,
                                     min_samples_leaf=50, random_state=rnd,
                                     max_leaf_nodes=n_peaks)
        x = np.arange(len(ts)).reshape((len(ts), 1))

        regr.fit(x, ts)
        x_test = np.arange(len(ts)).reshape((len(ts), 1))
        y_test = regr.predict(x_test)
        plt.plot(ts, '.k')
        plt.plot(x_test, y_test)

def max_pos(object, image):
    """
    Returns maximum position and widths in both direction.

    :param object:
        ``skimage.measure._regionprops._RegionProperties`` instance.
    :param image:
        Original image
    :return:
        Tuple of max position & 2 widths of region.
    """
    subimage = image[object.bbox[0]: object.bbox[2],
               object.bbox[1]: object.bbox[3]]
    indx = np.unravel_index(subimage.argmax(), subimage.shape)
    return (object.bbox[0] + indx[0], object.bbox[1] + indx[1]),\
            object.bbox[2] - object.bbox[0], object.bbox[3] - object.bbox[1]

# TODO: All search functions must returns instances of ``Candidate`` class
def search_candidates_clf(dsp, frb_clf=None, training_frac=0.01):
    """
    Search FRB using ML.
    :param dsp:
        Dynamical spectra.
    :param frb_clf: (optional)
        Instance of ``PulseClassifier``. If ``None`` then initialize one and
        train usign data supplied. (default: ``None``)
    :param training_frac: (optional)
        Fraction of time interval of dynamical spectra used for training
        classifier. (default: ``0.01``)
    :return:
    """
    from classification import PulseClassifier
    if frb_clf is None:
        frb_clf = PulseClassifier()
        frb_clf.create_train_sample(dsp, frac=training_frac)
        frb_clf.train()
    candidates = frb_clf.classify(dsp)
    return candidates

# TODO: All search functions must returns instances of ``Candidate`` class
def search_candidates(image, n_d_x, n_d_y, t_0, d_t, d_dm):

    a = image.copy()
    s = generate_binary_structure(2, 2)
    # Label image
    labeled_array, num_features = label(a, structure=s)
    # Find objects
    objects = find_objects(labeled_array)
    props = regionprops(labeled_array, intensity_image=image)
    # Container of object's properties
    _objects = np.empty(num_features, dtype=[('label', 'int'),
                                             ('dx', '<f8'),
                                             ('dy', '<f8'),
                                             ('max_pos', 'int',
                                              (2,))])

    labels = np.arange(num_features) + 1
    dx = [int(obj[1].stop - obj[1].start) for obj in objects]
    dy = [int(obj[0].stop - obj[0].start) for obj in objects]

    # Filling objects structured array
    _objects['label'] = labels
    _objects['dx'] = dx
    _objects['dy'] = dy
    # Classify objects
    _objects = _objects[np.logical_and(_objects['dy'] > n_d_y,
                                       _objects['dx'] > n_d_x)]
    # Fetch positions of only successfuly classified objects
    _objects['max_pos'] = maximum_position(image, labels=labeled_array,
                                           index=_objects['label'])
    _objects = _objects[np.lexsort((_objects['dx'], _objects['dy']))[::-1]]

    candidates = list()
    for _object in _objects:
        max_pos = _object['max_pos']
        candidate = Candidate(t_0 + max_pos[1] * TimeDelta(d_t, format='sec'),
                              max_pos[0] * float(d_dm))
        candidates.append(candidate)

    return candidates

# FIXME: ``skimage.filters.median`` use float images with ranges ``[-1, 1]``. I
# can scale original, use ``median`` and then scale back - it is much faster
# then mine
def create_ellipses(tdm_image, disk_size=5, threshold_perc=99.5,
                    statistic='mean', dilation_selem=np.ones((3, 3))):
    """
    Function that pre-process de-dispersed plane `t-DM` by creating
    characteristic inclined ellipses in places where FRB is sitting.

    :param tdm_image:
        2D numpy.ndarray  of `t-DM` plane.
    :param disk_size: (optional)
        Disk size to use when calculating filtered values. (default: ``5``)
    :param threshold_perc: (optional)
        Threshold [0 - 100] to threshold image after filtering. (default:
        ``99.5``)
    :param statistic: (optional)
        Statistic to use when filtering (``mean``, ``median`` or ``gauss``).
        (default: ``mean``)
    :param dilation_selem: (optional)
        The neighborhood expressed as a 2-D array of 1’s and 0’s for dilation
        step. (default: ``np.ones((3, 3))``)

    :return:
        2D numpy.ndarray of thresholded image of `t - DM` plane.
    """
    statistic_dict = {'mean': circular_mean, 'median': circular_median,
                      'gauss': gaussian_filter}
    image = tdm_image.copy()
    image = statistic_dict[statistic](image, disk_size)
    threshold = np.percentile(image.ravel(), threshold_perc)
    image[image < threshold] = 0
    image = dilation(image, dilation_selem)
    return image

def circular_mean(data, radius):
    """
    :param data:
    :param radius:
    :return:
    """
    from scipy.ndimage.filters import generic_filter as gf
    from skimage.morphology import disk

    kernel = disk(radius)
    return gf(data, np.mean, footprint=kernel)

def gaussian_filter(data, sigma):
    from skimage.filters import gaussian_filter as gf
    return gf(data, sigma)

def circular_median(data, radius):
    """
    :param data:
    :param radius:
    :return:
    """
    from scipy.ndimage.filters import generic_filter as gf
    from skimage.morphology import disk

    kernel = disk(radius)
    return gf(data, np.median, footprint=kernel)

def get_props(image, threshold):
    """
    Rerurn measured properties list of imaged labeled at specified threshold.

    :param image:
        Numpy 2D array with image.
    :param threshold:
        Threshold to label image. [0.-100.]

    :return:
        List of RegionProperties -
        (``skimage.measure._regionprops._RegionProperties`` instances)
    """
    threshold = np.percentile(image.ravel(), threshold)
    a = image.copy()
    # Keep only tail of image values distribution with signal
    a[a < threshold] = 0
    s = generate_binary_structure(2, 2)
    # Label image
    labeled_array, num_features = label(a, structure=s)
    return regionprops(labeled_array, intensity_image=image)

<commit_msg>- unused funcs for finding noise regions<commit_after># -*- coding: utf-8 -*-
import numpy as np
from scipy.ndimage.measurements import maximum_position, label, find_objects
from scipy.ndimage.morphology import generate_binary_structure
from skimage.measure import regionprops
from skimage.morphology import dilation
import matplotlib.pyplot as plt
from candidates import Candidate
from astropy.time import TimeDelta

def find_peaks(array_like, n_std=4, med_width=31, gauss_width=2):
    """
    Find peaks in 1D array.

    Data first median filtered with window size ``med_width``. Then it convolved
    with gaussian filter of width ``gauss_width``. Indexes of data with values
    higher then ``n_std`` are returned.

    :param array_like:
        Iterable of data values.
    :param med_width: (optional)
        Width of median filter to preprocess data. (default: ``30``)
    :param gauss_width: (optional)
        Width of gaussian filter to convolve median filtered data. (default:
        ``2``)
    :param n_std: (optional)
        Number of standard deviations to consider when searching peaks.
        (default: 4)

    :return:
        Numpy array of indexes of peak values.
    """
    import scipy
    array = np.asarray(array_like)
    array = scipy.signal.medfilt(array, med_width)
    garray = scipy.ndimage.filters.gaussian_filter1d(array, gauss_width)
    ind = (garray - np.mean(garray)) > n_std * np.std(garray)
    return ind

def find_noisy(dsp, n, axis=0, rnd=42):
    """
    Attempt to find noisy parts of dynamical spectra. Fitting gaussian mixture
    model to histogram of ``dsp`` value. If 2 components is superior model based
    on BIC then we can find typical threshold for high-amplitude noise.
    :param dsp:
    :param n:
    :param axis:
    :return:
    """
    from sklearn.mixture import GMM
    results = dict()
    data = dsp.reshape((dsp.size, 1))
    for i in range(1, n + 1):
        classif = GMM(n_components=i)
        classif.fit(data)
        results.update({classif.bic(data): [i, classif]})
    min_bic = min(results.keys())
    i, clf = results[min_bic]
    if i == 2:
        threshold = (clf.means_ + 3. * np.sqrt(clf.covars_))[0][0]
        ts = np.mean(dsp, axis=0)
        import scipy
        tsf = scipy.signal.medfilt(ts, 3)
        dsp_changes = tsf < threshold
        n_peaks = len(np.where(dsp_changes[:-1] != dsp_changes[1:])[0])
        regr = DecisionTreeRegressor(max_depth=n_peaks, min_samples_split=50,
                                     min_samples_leaf=50, random_state=rnd,
                                     max_leaf_nodes=n_peaks)
        x = np.arange(len(ts)).reshape((len(ts), 1))

        regr.fit(x, ts)
        x_test = np.arange(len(ts)).reshape((len(ts), 1))
        y_test = regr.predict(x_test)
        plt.plot(ts, '.k')
        plt.plot(x_test, y_test)

def max_pos(object, image):
    """
    Returns maximum position and widths in both direction.

    :param object:
        ``skimage.measure._regionprops._RegionProperties`` instance.
    :param image:
        Original image
    :return:
        Tuple of max position & 2 widths of region.
    """
    subimage = image[object.bbox[0]: object.bbox[2],
               object.bbox[1]: object.bbox[3]]
    indx = np.unravel_index(subimage.argmax(), subimage.shape)
    return (object.bbox[0] + indx[0], object.bbox[1] + indx[1]),\
            object.bbox[2] - object.bbox[0], object.bbox[3] - object.bbox[1]

# TODO: All search functions must returns instances of ``Candidate`` class
def search_candidates_clf(dsp, frb_clf=None, training_frac=0.01):
    """
    Search FRB using ML.
    :param dsp:
        Dynamical spectra.
    :param frb_clf: (optional)
        Instance of ``PulseClassifier``. If ``None`` then initialize one and
        train usign data supplied. (default: ``None``)
    :param training_frac: (optional)
        Fraction of time interval of dynamical spectra used for training
        classifier. (default: ``0.01``)
    :return:
    """
    from classification import PulseClassifier
    if frb_clf is None:
        frb_clf = PulseClassifier()
        frb_clf.create_train_sample(dsp, frac=training_frac)
        frb_clf.train()
    candidates = frb_clf.classify(dsp)
    return candidates

# TODO: All search functions must returns instances of ``Candidate`` class
def search_candidates(image, n_d_x, n_d_y, t_0, d_t, d_dm):

    a = image.copy()
    s = generate_binary_structure(2, 2)
    # Label image
    labeled_array, num_features = label(a, structure=s)
    # Find objects
    objects = find_objects(labeled_array)
    props = regionprops(labeled_array, intensity_image=image)
    # Container of object's properties
    _objects = np.empty(num_features, dtype=[('label', 'int'),
                                             ('dx', '<f8'),
                                             ('dy', '<f8'),
                                             ('max_pos', 'int',
                                              (2,))])

    labels = np.arange(num_features) + 1
    dx = [int(obj[1].stop - obj[1].start) for obj in objects]
    dy = [int(obj[0].stop - obj[0].start) for obj in objects]

    # Filling objects structured array
    _objects['label'] = labels
    _objects['dx'] = dx
    _objects['dy'] = dy
    # Classify objects
    _objects = _objects[np.logical_and(_objects['dy'] > n_d_y,
                                       _objects['dx'] > n_d_x)]
    # Fetch positions of only successfuly classified objects
    _objects['max_pos'] = maximum_position(image, labels=labeled_array,
                                           index=_objects['label'])
    _objects = _objects[np.lexsort((_objects['dx'], _objects['dy']))[::-1]]

    candidates = list()
    for _object in _objects:
        max_pos = _object['max_pos']
        candidate = Candidate(t_0 + max_pos[1] * TimeDelta(d_t, format='sec'),
                              max_pos[0] * float(d_dm))
        candidates.append(candidate)

    return candidates

# FIXME: ``skimage.filters.median`` use float images with ranges ``[-1, 1]``. I
# can scale original, use ``median`` and then scale back - it is much faster
# then mine
def create_ellipses(tdm_image, disk_size=5, threshold_perc=99.5,
                    statistic='mean', dilation_selem=np.ones((3, 3))):
    """
    Function that pre-process de-dispersed plane `t-DM` by creating
    characteristic inclined ellipses in places where FRB is sitting.

    :param tdm_image:
        2D numpy.ndarray  of `t-DM` plane.
    :param disk_size: (optional)
        Disk size to use when calculating filtered values. (default: ``5``)
    :param threshold_perc: (optional)
        Threshold [0 - 100] to threshold image after filtering. (default:
        ``99.5``)
    :param statistic: (optional)
        Statistic to use when filtering (``mean``, ``median`` or ``gauss``).
        (default: ``mean``)
    :param dilation_selem: (optional)
        The neighborhood expressed as a 2-D array of 1’s and 0’s for dilation
        step. (default: ``np.ones((3, 3))``)

    :return:
        2D numpy.ndarray of thresholded image of `t - DM` plane.
    """
    statistic_dict = {'mean': circular_mean, 'median': circular_median,
                      'gauss': gaussian_filter}
    image = tdm_image.copy()
    image = statistic_dict[statistic](image, disk_size)
    threshold = np.percentile(image.ravel(), threshold_perc)
    image[image < threshold] = 0
    image = dilation(image, dilation_selem)
    return image

def circular_mean(data, radius):
    """
    :param data:
    :param radius:
    :return:
    """
    from scipy.ndimage.filters import generic_filter as gf
    from skimage.morphology import disk

    kernel = disk(radius)
    return gf(data, np.mean, footprint=kernel)

def gaussian_filter(data, sigma):
    from skimage.filters import gaussian_filter as gf
    return gf(data, sigma)

def circular_median(data, radius):
    """
    :param data:
    :param radius:
    :return:
    """
    from scipy.ndimage.filters import generic_filter as gf
    from skimage.morphology import disk

    kernel = disk(radius)
    return gf(data, np.median, footprint=kernel)

def get_props(image, threshold):
    """
    Rerurn measured properties list of imaged labeled at specified threshold.

    :param image:
        Numpy 2D array with image.
    :param threshold:
        Threshold to label image. [0.-100.]

    :return:
        List of RegionProperties -
        (``skimage.measure._regionprops._RegionProperties`` instances)
    """
    threshold = np.percentile(image.ravel(), threshold)
    a = image.copy()
    # Keep only tail of image values distribution with signal
    a[a < threshold] = 0
    s = generate_binary_structure(2, 2)
    # Label image
    labeled_array, num_features = label(a, structure=s)
    return regionprops(labeled_array, intensity_image=image)

<|endoftext|>


<commit_before>#3D Reconstruct from a tilt series using Direct Fourier Method
#
#NOTE: Tilt Axis needs to be in the third dimension
#
#developed as part of the tomviz project (www.tomviz.com)

def transform_scalars(dataset):
    from tomviz import utils
    import numpy as np

    #----USER SPECIFIED VARIABLES-----#
    TILT_AXIS  = []  #Specify the tilt axis, if none is specified
                     #it assume it is the smallest dimension
    ANGLES = []  #Specify the angles used in the tiltseries
                 #For example, ANGLES = np.linspace(0,140,70)
    #---------------------------------#

    data_py = utils.get_array(dataset)
    if data_py is None:
        raise RuntimeError("No scalars found!")

    if TILT_AXIS == []: #If tilt axis is not given, find it
    #Find smallest array dimension, assume it is the tilt angle axis
        if data_py.ndim == 3:
            TILT_AXIS = np.argmin( data_py.shape )
        elif data_py.ndim == 2:
            raise RuntimeError("Data Array is 2 dimensions, it should be 3!")
        else:
            raise RuntimeError("Data Array is not 2 or 3 dimensions!")

    if ANGLES == []: #If angles are not given, assume 2 degree tilts
        angles = np.linspace(0,146,74)

    dimx = np.size(data_py,0) #IN THE FUTURE THIS SHOULD NOT BE ASSUMED
    result3D = dfm3(data_py,angles,dimx)

    # set the result as the new scalars.
    utils.set_array(dataset, result3D)
    print('Reconsruction Complete')

#dfm3(input,angles,Npad)
#3D Direct Fourier Method
#Use numpy fft
#
#Yi Jiang
#08/25/2014
#input: aligned data
#angles: projection angles
#N_pad: size of padded projection. typical choice: twice as the original projection

import numpy as np

def dfm3(input,angles,Npad):
    print "You are using 3D direct fourier method, be patient."
    input = np.double(input)
    (Nx,Ny,Np) = input.shape
    angles = np.double(angles)
    cen = np.floor(Ny/2)
    cen_pad = np.floor(Npad/2)
    pad_post = np.ceil((Npad-Ny)/2); pad_pre = np.floor((Npad-Ny)/2)
    #initialize value and weight matrix
    print "initialization"
    Nz = Ny
    w = np.zeros((Nx,Ny,Nz))
    v = np.zeros((Nx,Ny,Nz)) + 1j*np.zeros((Nx,Ny,Nz))

    dk = np.double(Ny)/np.double(Npad) *1

    for a in range(0,Np):
        print angles[a]
        ang = angles[a]*np.pi/180
	projection = input[:,:,a] #projection
        p = np.lib.pad(projection,((0,0),(pad_pre,pad_post)),'constant',constant_values=(0,0)) #pad zeros
        p = np.fft.ifftshift(p)
	pF = np.fft.fft2(p)
	pF = np.fft.fftshift(pF)

        #bilinear extrapolation
        for i in range(0, np.int(Npad)):
            ky = (i-cen_pad)*dk;  #kz = 0;
            ky_new = np.cos(ang)*ky #new coord. after rotation
            kz_new = np.sin(ang)*ky
	    sy = abs(np.floor(ky_new) - ky_new) #calculate weights
            sz = abs(np.floor(kz_new) - kz_new)
            for b in range(1,5): #bilinear extrapolation
                pz,py,weight = bilinear(kz_new,ky_new,sz,sy,Ny,b)
		pz = pz + cen;
		py = py + cen;
                if (py>=0 and py<Ny and pz>=0 and pz<Nz):
                   w[:,py,pz] = w[:,py,pz] + weight
	           v[:,py,pz] = v[:,py,pz] + weight * pF[:,i]
    v[w!=0] = v[w!=0]/w[w!=0]
    v = np.fft.ifftshift(v)
    recon = np.real(np.fft.ifftn(v))
    recon = np.fft.fftshift(recon)
    return recon.astype(np.float32)

def bilinear(kz_new,ky_new,sz,sy,N,p):
    if p==1:
       py = np.floor(ky_new)
       pz = np.floor(kz_new)
       weight = (1-sy)*(1-sz)
    elif p==2:
       py = np.ceil(ky_new) #P2
       pz = np.floor(kz_new)
       weight = sy*(1-sz)
    elif p==3:
       py = np.floor(ky_new) #P3
       pz = np.ceil(kz_new)
       weight = (1-sy)*sz
    elif p==4:
       py = np.ceil(ky_new) #P4
       pz = np.ceil(kz_new)
       weight = sy*sz
    if py<0:
       py = N+py
    else:
       py = py
    return (pz,py,weight)

<commit_msg>Improved Recon_DFT.py<commit_after>def transform_scalars(dataset):
    """3D Reconstruct from a tilt series using Direct Fourier Method"""

    from tomviz import utils
    import numpy as np

    try: #If dataset is marked as tilt series
        tilt_angles = utils.get_tilt_angles(dataset) #Get Tilt angles
        data_py = utils.get_array(dataset)
        if data_py is None:
           raise RuntimeError("No scalars found!")

        recon = dfm3(data_py,angles,np.size(data_py,0))
        print('Reconsruction Complete')

        # set the result as the new scalars.
        utils.set_array(dataset, recon)

        # Mark dataset as tilt series
        utils.mark_as_volume(dataset)

    except:
         pass

import numpy as np
def dfm3(input,angles,Npad):
    #input: aligned data
    #angles: projection angles
    #N_pad: size of padded projection. typical choice: twice as the original projection

    input = np.double(input)
    (Nx,Ny,Np) = input.shape
    angles = np.double(angles)
    cen = np.floor(Ny/2)
    cen_pad = np.floor(Npad/2)
    pad_post = np.ceil((Npad-Ny)/2); pad_pre = np.floor((Npad-Ny)/2)
    #initialize value and weight matrix
    Nz = Ny
    w = np.zeros((Nx,Ny,Nz))
    v = np.zeros((Nx,Ny,Nz)) + 1j*np.zeros((Nx,Ny,Nz))

    dk = np.double(Ny)/np.double(Npad) *1

    for a in range(0,Np):
        print angles[a]
        ang = angles[a]*np.pi/180
	projection = input[:,:,a] #projection
        p = np.lib.pad(projection,((0,0),(pad_pre,pad_post)),'constant',constant_values=(0,0)) #pad zeros
        p = np.fft.ifftshift(p)
	pF = np.fft.fft2(p)
	pF = np.fft.fftshift(pF)

        #bilinear extrapolation
        for i in range(0, np.int(Npad)):
            ky = (i-cen_pad)*dk;  #kz = 0;
            ky_new = np.cos(ang)*ky #new coord. after rotation
            kz_new = np.sin(ang)*ky
	    sy = abs(np.floor(ky_new) - ky_new) #calculate weights
            sz = abs(np.floor(kz_new) - kz_new)
            for b in range(1,5): #bilinear extrapolation
                pz,py,weight = bilinear(kz_new,ky_new,sz,sy,Ny,b)
		pz = pz + cen;
		py = py + cen;
                if (py>=0 and py<Ny and pz>=0 and pz<Nz):
                   w[:,py,pz] = w[:,py,pz] + weight
	           v[:,py,pz] = v[:,py,pz] + weight * pF[:,i]
    v[w!=0] = v[w!=0]/w[w!=0]
    v = np.fft.ifftshift(v)
    recon = np.real(np.fft.ifftn(v))
    recon = np.fft.fftshift(recon)
    return recon.astype(np.float32)

def bilinear(kz_new,ky_new,sz,sy,N,p):
    if p==1:
       py = np.floor(ky_new)
       pz = np.floor(kz_new)
       weight = (1-sy)*(1-sz)
    elif p==2:
       py = np.ceil(ky_new) #P2
       pz = np.floor(kz_new)
       weight = sy*(1-sz)
    elif p==3:
       py = np.floor(ky_new) #P3
       pz = np.ceil(kz_new)
       weight = (1-sy)*sz
    elif p==4:
       py = np.ceil(ky_new) #P4
       pz = np.ceil(kz_new)
       weight = sy*sz
    if py<0:
       py = N+py
    else:
       py = py
    return (pz,py,weight)

<|endoftext|>


<commit_before>from django.conf.urls import include, url
from django.urls import reverse
from django.conf.urls.static import static
from django.conf import settings
from django.http import HttpResponseRedirect
from django.contrib.staticfiles.urls import staticfiles_urlpatterns
from django.contrib.sitemaps import views as sitemaps_views, Sitemap
from django.utils.translation import pgettext
from django.contrib import admin

from rest_framework.schemas import get_schema_view

from froide.account.api_views import ProfileView
from froide.foirequest.api_views import (FoiRequestViewSet, FoiMessageViewSet,
    FoiAttachmentViewSet)
from froide.publicbody.api_views import (ClassificationViewSet,
    CategoryViewSet, PublicBodyViewSet, JurisdictionViewSet, FoiLawViewSet)
from froide.georegion.api_views import GeoRegionViewSet
from froide.foirequestfollower.api_views import FoiRequestFollowerViewSet
from froide.campaign.api_views import CampaignViewSet
from froide.upload.api_views import UploadViewSet
from froide.document.api_views import (
    PageViewSet, DocumentViewSet, DocumentCollectionViewSet,
    PageAnnotationViewSet
)
from froide.document.urls import document_media_urlpatterns

from froide.publicbody.views import (PublicBodySitemap, FoiLawSitemap,
                                     JurisdictionSitemap, show_publicbody,
                                     publicbody_shortlink)
from froide.foirequest.views import FoiRequestSitemap, index, dashboard

from froide.helper import api_router

def handler500(request):
    """
    500 error handler which includes ``request`` in the context.
    """

    from django.shortcuts import render
    return render(request, '500.html', {'request': request}, status=500)

api_router.register(r'request', FoiRequestViewSet, basename='request')
api_router.register(r'message', FoiMessageViewSet, basename='message')
api_router.register(r'attachment', FoiAttachmentViewSet,
                    basename='attachment')
api_router.register(r'publicbody', PublicBodyViewSet, basename='publicbody')
api_router.register(r'category', CategoryViewSet, basename='category')
api_router.register(r'classification', ClassificationViewSet,
                    basename='classification')
api_router.register(r'jurisdiction', JurisdictionViewSet,
                    basename='jurisdiction')
api_router.register(r'law', FoiLawViewSet, basename='law')
api_router.register(r'georegion', GeoRegionViewSet, basename='georegion')
api_router.register(r'following', FoiRequestFollowerViewSet, basename='following')
api_router.register(r'campaign', CampaignViewSet, basename='campaign')
api_router.register(r'upload', UploadViewSet, basename='upload')
api_router.register(r'document', DocumentViewSet, basename='document')
api_router.register(r'documentcollection',
    DocumentCollectionViewSet, basename='documentcollection')
api_router.register(r'page', PageViewSet, basename='page')
api_router.register(r'pageannotation', PageAnnotationViewSet,
    basename='pageannotation')

class StaticViewSitemap(Sitemap):
    priority = 1.0
    changefreq = 'daily'

    def items(self):
        return ['foirequest-list']

    def location(self, item):
        return reverse(item)

sitemaps = {
    'publicbody': PublicBodySitemap,
    'foilaw': FoiLawSitemap,
    'jurisdiction': JurisdictionSitemap,
    'foirequest': FoiRequestSitemap,
    'content': StaticViewSitemap,
}

PROTOCOL = settings.SITE_URL.split(':')[0]

for klass in sitemaps.values():
    klass.protocol = PROTOCOL

sitemap_urlpatterns = [
    url(r'^sitemap\.xml$', sitemaps_views.index,
        {'sitemaps': sitemaps, 'sitemap_url_name': 'sitemaps'}),
    url(r'^sitemap-(?P<section>.+)\.xml$', sitemaps_views.sitemap,
        {'sitemaps': sitemaps}, name='sitemaps')
]

froide_urlpatterns = []

SECRET_URLS = getattr(settings, "SECRET_URLS", {})

if settings.FROIDE_CONFIG.get('api_activated', True):
    schema_view = get_schema_view(title='{name} API'.format(
                                  name=settings.SITE_NAME))
    froide_urlpatterns += [
        url(r'^api/v1/user/', ProfileView.as_view(), name='api-user-profile'),
        url(r'^api/v1/', include((api_router.urls, 'api'))),
        url(r'^api/v1/schema/$', schema_view),
    ]

froide_urlpatterns += [
    # Translators: URL part
    url(r'^', include('froide.foirequest.urls')),
]

if len(settings.LANGUAGES) > 1:
    froide_urlpatterns += [
        url(r'^i18n/', include('django.conf.urls.i18n'))
    ]

account = pgettext('url part', 'account')
documents = pgettext('url part', 'documents')
teams = pgettext('url part', 'teams')

froide_urlpatterns += [
    # Translators: follow request URL
    url(r'^%s/' % pgettext('url part', 'follow'), include('froide.foirequestfollower.urls')),

    url(r"^%s/(?P<obj_id>\d+)/?$" % pgettext('url part', 'b'),
        publicbody_shortlink, name="publicbody-publicbody_shortlink"),
    # Translators: URL part
    url(r"^%s/(?P<slug>[-\w]+)/$" % pgettext('url part', 'entity'), show_publicbody,
            name="publicbody-show"),
    url(r"^%s/$" % pgettext('url part', 'entity'),
        lambda request: HttpResponseRedirect(reverse('publicbody-list'))),
    # Translators: URL part
    url(r'^%s/' % pgettext('url part', 'entities'), include('froide.publicbody.urls')),
    # Translators: URL part
    url(r'^%s/' % pgettext('url part', 'law'), include('froide.publicbody.law_urls')),
    url(r'^%s/' % documents, include('froide.document.urls')),
    url(r'^%s/%s/' % (account, teams), include('froide.team.urls')),
    url(r'^%s/' % account, include('froide.account.urls')),
    url(r'^%s/access-token/' % account, include('froide.accesstoken.urls')),
    # Translators: URL part
    url(r'^%s/' % pgettext('url part', 'profile'), include('froide.account.profile_urls')),
    url(r'^comments/', include('django_comments.urls')),
    url(r'^problem/', include('froide.problem.urls')),
    url(r'^guide/', include('froide.guide.urls')),
]

froide_urlpatterns += document_media_urlpatterns

admin_urls = [
        url(r'^%s/' % SECRET_URLS.get('admin', 'admin'), admin.site.urls)
]

if SECRET_URLS.get('postmark_inbound'):
    from froide.foirequest.views import postmark_inbound

    froide_urlpatterns += [
        url(r'^postmark/%s/' % SECRET_URLS['postmark_inbound'],
            postmark_inbound, name="foirequest-postmark_inbound")
    ]

if SECRET_URLS.get('postmark_bounce'):
    from froide.foirequest.views import postmark_bounce

    froide_urlpatterns += [
        url(r'^postmark/%s/' % SECRET_URLS['postmark_bounce'],
            postmark_bounce, name="foirequest-postmark_bounce")
    ]

if settings.DEBUG:
    froide_urlpatterns += staticfiles_urlpatterns()
    froide_urlpatterns += static(
        settings.MEDIA_URL,
        document_root=settings.MEDIA_ROOT
    )

if settings.DEBUG:
    try:
        import debug_toolbar
        froide_urlpatterns = [
            url(r'^__debug__/', include(debug_toolbar.urls)),
        ] + froide_urlpatterns
    except ImportError:
        pass

# Translators: URL part
jurisdiction_part = pgettext('url part', 'jurisdiction')

jurisdiction_urls = [
    url(r'^%s/(?P<slug>[\w-]+)/' % jurisdiction_part,
        include('froide.publicbody.jurisdiction_urls'))
]

urlpatterns = froide_urlpatterns + [
    url(r'^$', index, name='index'),
    url(r'^dashboard/$', dashboard, name='dashboard'),
] + sitemap_urlpatterns + jurisdiction_urls + admin_urls
<commit_msg>Add 500 error page URL for demo purposes<commit_after>from django.conf.urls import include, url
from django.urls import reverse
from django.conf.urls.static import static
from django.conf import settings
from django.http import HttpResponseRedirect
from django.contrib.staticfiles.urls import staticfiles_urlpatterns
from django.contrib.sitemaps import views as sitemaps_views, Sitemap
from django.utils.translation import pgettext
from django.contrib import admin
from django.template.response import TemplateResponse

from rest_framework.schemas import get_schema_view

from froide.account.api_views import ProfileView
from froide.foirequest.api_views import (FoiRequestViewSet, FoiMessageViewSet,
    FoiAttachmentViewSet)
from froide.publicbody.api_views import (ClassificationViewSet,
    CategoryViewSet, PublicBodyViewSet, JurisdictionViewSet, FoiLawViewSet)
from froide.georegion.api_views import GeoRegionViewSet
from froide.foirequestfollower.api_views import FoiRequestFollowerViewSet
from froide.campaign.api_views import CampaignViewSet
from froide.upload.api_views import UploadViewSet
from froide.document.api_views import (
    PageViewSet, DocumentViewSet, DocumentCollectionViewSet,
    PageAnnotationViewSet
)
from froide.document.urls import document_media_urlpatterns

from froide.publicbody.views import (PublicBodySitemap, FoiLawSitemap,
                                     JurisdictionSitemap, show_publicbody,
                                     publicbody_shortlink)
from froide.foirequest.views import FoiRequestSitemap, index, dashboard

from froide.helper import api_router

def handler500(request):
    """
    500 error handler which includes ``request`` in the context.
    """

    from django.shortcuts import render
    return render(request, '500.html', {'request': request}, status=500)

api_router.register(r'request', FoiRequestViewSet, basename='request')
api_router.register(r'message', FoiMessageViewSet, basename='message')
api_router.register(r'attachment', FoiAttachmentViewSet,
                    basename='attachment')
api_router.register(r'publicbody', PublicBodyViewSet, basename='publicbody')
api_router.register(r'category', CategoryViewSet, basename='category')
api_router.register(r'classification', ClassificationViewSet,
                    basename='classification')
api_router.register(r'jurisdiction', JurisdictionViewSet,
                    basename='jurisdiction')
api_router.register(r'law', FoiLawViewSet, basename='law')
api_router.register(r'georegion', GeoRegionViewSet, basename='georegion')
api_router.register(r'following', FoiRequestFollowerViewSet, basename='following')
api_router.register(r'campaign', CampaignViewSet, basename='campaign')
api_router.register(r'upload', UploadViewSet, basename='upload')
api_router.register(r'document', DocumentViewSet, basename='document')
api_router.register(r'documentcollection',
    DocumentCollectionViewSet, basename='documentcollection')
api_router.register(r'page', PageViewSet, basename='page')
api_router.register(r'pageannotation', PageAnnotationViewSet,
    basename='pageannotation')

class StaticViewSitemap(Sitemap):
    priority = 1.0
    changefreq = 'daily'

    def items(self):
        return ['foirequest-list']

    def location(self, item):
        return reverse(item)

sitemaps = {
    'publicbody': PublicBodySitemap,
    'foilaw': FoiLawSitemap,
    'jurisdiction': JurisdictionSitemap,
    'foirequest': FoiRequestSitemap,
    'content': StaticViewSitemap,
}

PROTOCOL = settings.SITE_URL.split(':')[0]

for klass in sitemaps.values():
    klass.protocol = PROTOCOL

sitemap_urlpatterns = [
    url(r'^sitemap\.xml$', sitemaps_views.index,
        {'sitemaps': sitemaps, 'sitemap_url_name': 'sitemaps'}),
    url(r'^sitemap-(?P<section>.+)\.xml$', sitemaps_views.sitemap,
        {'sitemaps': sitemaps}, name='sitemaps')
]

froide_urlpatterns = []

SECRET_URLS = getattr(settings, "SECRET_URLS", {})

if settings.FROIDE_CONFIG.get('api_activated', True):
    schema_view = get_schema_view(title='{name} API'.format(
                                  name=settings.SITE_NAME))
    froide_urlpatterns += [
        url(r'^api/v1/user/', ProfileView.as_view(), name='api-user-profile'),
        url(r'^api/v1/', include((api_router.urls, 'api'))),
        url(r'^api/v1/schema/$', schema_view),
    ]

froide_urlpatterns += [
    # Translators: URL part
    url(r'^', include('froide.foirequest.urls')),
]

if len(settings.LANGUAGES) > 1:
    froide_urlpatterns += [
        url(r'^i18n/', include('django.conf.urls.i18n'))
    ]

account = pgettext('url part', 'account')
documents = pgettext('url part', 'documents')
teams = pgettext('url part', 'teams')

froide_urlpatterns += [
    # Translators: follow request URL
    url(r'^%s/' % pgettext('url part', 'follow'), include('froide.foirequestfollower.urls')),

    url(r"^%s/(?P<obj_id>\d+)/?$" % pgettext('url part', 'b'),
        publicbody_shortlink, name="publicbody-publicbody_shortlink"),
    # Translators: URL part
    url(r"^%s/(?P<slug>[-\w]+)/$" % pgettext('url part', 'entity'), show_publicbody,
            name="publicbody-show"),
    url(r"^%s/$" % pgettext('url part', 'entity'),
        lambda request: HttpResponseRedirect(reverse('publicbody-list'))),
    # Translators: URL part
    url(r'^%s/' % pgettext('url part', 'entities'), include('froide.publicbody.urls')),
    # Translators: URL part
    url(r'^%s/' % pgettext('url part', 'law'), include('froide.publicbody.law_urls')),
    url(r'^%s/' % documents, include('froide.document.urls')),
    url(r'^%s/%s/' % (account, teams), include('froide.team.urls')),
    url(r'^%s/' % account, include('froide.account.urls')),
    url(r'^%s/access-token/' % account, include('froide.accesstoken.urls')),
    # Translators: URL part
    url(r'^%s/' % pgettext('url part', 'profile'), include('froide.account.profile_urls')),
    url(r'^comments/', include('django_comments.urls')),
    url(r'^problem/', include('froide.problem.urls')),
    url(r'^guide/', include('froide.guide.urls')),
    url(r'^500/', lambda request: TemplateResponse(request, '500.html'))
]

froide_urlpatterns += document_media_urlpatterns

admin_urls = [
        url(r'^%s/' % SECRET_URLS.get('admin', 'admin'), admin.site.urls)
]

if SECRET_URLS.get('postmark_inbound'):
    from froide.foirequest.views import postmark_inbound

    froide_urlpatterns += [
        url(r'^postmark/%s/' % SECRET_URLS['postmark_inbound'],
            postmark_inbound, name="foirequest-postmark_inbound")
    ]

if SECRET_URLS.get('postmark_bounce'):
    from froide.foirequest.views import postmark_bounce

    froide_urlpatterns += [
        url(r'^postmark/%s/' % SECRET_URLS['postmark_bounce'],
            postmark_bounce, name="foirequest-postmark_bounce")
    ]

if settings.DEBUG:
    froide_urlpatterns += staticfiles_urlpatterns()
    froide_urlpatterns += static(
        settings.MEDIA_URL,
        document_root=settings.MEDIA_ROOT
    )

if settings.DEBUG:
    try:
        import debug_toolbar
        froide_urlpatterns = [
            url(r'^__debug__/', include(debug_toolbar.urls)),
        ] + froide_urlpatterns
    except ImportError:
        pass

# Translators: URL part
jurisdiction_part = pgettext('url part', 'jurisdiction')

jurisdiction_urls = [
    url(r'^%s/(?P<slug>[\w-]+)/' % jurisdiction_part,
        include('froide.publicbody.jurisdiction_urls'))
]

urlpatterns = froide_urlpatterns + [
    url(r'^$', index, name='index'),
    url(r'^dashboard/$', dashboard, name='dashboard'),
] + sitemap_urlpatterns + jurisdiction_urls + admin_urls
<|endoftext|>


<commit_before>"""
Created on April 5, 2016
@author: maljdp
extracted from alfoa (11/14/2013) OutStreamManager.py
"""
#for future compatibility with Python 3-----------------------------------------
from __future__ import division, print_function, unicode_literals, absolute_import
import warnings
warnings.simplefilter('default',DeprecationWarning)
if not 'xrange' in dir(__builtins__):
  xrange = range
#End compatibility block for Python 3-------------------------------------------

################################################################################
from .OutStreamManager import OutStreamManager
from .OutStreamPlot import OutStreamPlot
from .OutStreamPrint import OutStreamPrint
## [ Add new class here ]
################################################################################
## Alternatively, to fully automate this file:
# from OutStreamManagers import *
################################################################################

"""
 Interface Dictionary (factory) (private)
"""
# This machinery will automatically populate the "knownTypes" given the
# imports defined above.
__base = 'OutStreamManager'
__interFaceDict = {}

for classObj in eval(__base).__subclasses__():
  ## As long as these subclasses follow the pattern of starting with OutStream
  ## this will appropriately key them to a more user-friendly name without the
  ## need for them to redudantly prepend "X" as "OutStreamX"
  key = classObj.__name__.replace('OutStream','')
  __interFaceDict[key] = classObj

def knownTypes():
  """
  Returns a list of strings that define the types of instantiable objects for
  this base factory.
  """
  return __interFaceDict.keys()

def returnInstance(Type,caller):
  """
  Attempts to create and return an instance of a particular type of object
  available to this factory.
  @ In, Type, string should be one of the knownTypes.
  @ In, caller, the object requesting the instance
                (used for error/debug messaging).
  @ Out, subclass object constructed with no arguments
  """
  try:
    return __interFaceDict[Type]()
  except KeyError:
    print(eval(__base).__subclasses__())
    print(__interfaceDict.keys())
    caller.raiseAnError(NameError,__name__+': unknown '+__base+' type '+Type)

def returnClass(Type,caller):
  """
  Attempts to return a particular class type available to this factory.
  @ In, Type, string should be one of the knownTypes.
  @ In, caller, the object requesting the class
                (used for error/debug messaging).
  @ Out, reference to the subclass
  """
  try:
    return __interFaceDict[Type]
  except KeyError:
    caller.raiseAnError(NameError,__name__+': unknown '+__base+' type '+Type)
<commit_msg>Fixing docstrings as per alfoa's request.<commit_after>"""
  Created on April 5, 2016
  @author: maljdp
  extracted from alfoa (11/14/2013) OutStreamManager.py
"""
#for future compatibility with Python 3-----------------------------------------
from __future__ import division, print_function, unicode_literals, absolute_import
import warnings
warnings.simplefilter('default',DeprecationWarning)
if not 'xrange' in dir(__builtins__):
  xrange = range
#End compatibility block for Python 3-------------------------------------------

################################################################################
from .OutStreamManager import OutStreamManager
from .OutStreamPlot import OutStreamPlot
from .OutStreamPrint import OutStreamPrint
## [ Add new class here ]
################################################################################
## Alternatively, to fully automate this file:
# from OutStreamManagers import *
################################################################################

"""
  Interface Dictionary (factory) (private)
"""
# This machinery will automatically populate the "knownTypes" given the
# imports defined above.
__base = 'OutStreamManager'
__interFaceDict = {}

for classObj in eval(__base).__subclasses__():
  ## As long as these subclasses follow the pattern of starting with OutStream
  ## this will appropriately key them to a more user-friendly name without the
  ## need for them to redudantly prepend "X" as "OutStreamX"
  key = classObj.__name__.replace('OutStream','')
  __interFaceDict[key] = classObj

def knownTypes():
  """
    Returns a list of strings that define the types of instantiable objects for
    this base factory.
  """
  return __interFaceDict.keys()

def returnInstance(Type,caller):
  """
    Attempts to create and return an instance of a particular type of object
    available to this factory.
    @ In, Type, string should be one of the knownTypes.
    @ In, caller, MessageUser, the object requesting the instance
                  (used for error/debug messaging).
    @ Out, __interFaceDict[Type], instance of OutStreamManager subclass,
      a subclass object constructed with no arguments
  """
  try:
    return __interFaceDict[Type]()
  except KeyError:
    print(eval(__base).__subclasses__())
    print(__interfaceDict.keys())
    caller.raiseAnError(NameError,__name__+': unknown '+__base+' type '+Type)

def returnClass(Type,caller):
  """
    Attempts to return a particular class type available to this factory.
    @ In, Type, string should be one of the knownTypes.
    @ In, caller, the object requesting the class
                  (used for error/debug messaging).
    @ Out, reference to the subclass
  """
  try:
    return __interFaceDict[Type]
  except KeyError:
    caller.raiseAnError(NameError,__name__+': unknown '+__base+' type '+Type)
<|endoftext|>


<commit_before># Generated by Django 2.2.5 on 2019-11-16 08:40

from django.db import migrations, models

class Migration(migrations.Migration):

    dependencies = [
        ('stock', '0015_auto_20190913_1407'),
    ]

    operations = [
        migrations.AddField(
            model_name='stockitem',
            name='active',
            field=models.BooleanField(default=True),
        ),
    ]
<commit_msg>Removed old migration file<commit_after><|endoftext|>


<commit_before>#! /usr/bin/env python
#
# Copyright (c) 2011 SEOmoz
#
# Permission is hereby granted, free of charge, to any person obtaining
# a copy of this software and associated documentation files (the
# "Software"), to deal in the Software without restriction, including
# without limitation the rights to use, copy, modify, merge, publish,
# distribute, sublicense, and/or sell copies of the Software, and to
# permit persons to whom the Software is furnished to do so, subject to
# the following conditions:
#
# The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

import re
import os
from sauron import logger
from sauron.metrics import Metric, MetricException

class LogMetric(Metric):
    def __init__(self, name, path, **kwargs):
        Metric.__init__(self, name, **kwargs)
        self.reconfig(name, path, **kwargs)

    def reconfig(self, name, path, **kwargs):
        Metric.reconfig(self, **kwargs)
        self.name = name
        self.patterns = dict([(k, re.compile(v)) for k,v in kwargs.items()])
        self.path = path
        try:
            self.f = file(self.path)    # The file object we'll read from
        except IOError as e:
            raise MetricException(e)
        self.stat = os.lstat(self.path) # The stats on that particular file

    def __del__(self):
        try:
            self.f.close()
        except ValueError:
            pass
        except AttributeError:
            pass

    def values(self):
        try:
            # Alright, first get new stats on the file
            s = os.lstat(self.path)
            # The lines we've read
            lines = []
            # Now, see if the file was nuked
            # I'm not sure how this works. Checking inode might not really capture
            # what we're talking about. It certainly happens when the file is replaced,
            # but there /may/ be other times when it changes
            if s.st_ino != self.stat.st_ino:
                logger.warn('Inode for %s has changed' % self.path)
                self.f.close()
                self.f = file(self.path)
                lines = self.f.read(s.st_size).strip().split('\n')
            elif s.st_size < self.stat.st_size:
                logger.warn('File %s has shrunk since last read! Reading from beginning...' % self.path)
                self.f.seek(0)
                lines = self.f.read(s.st_size).strip().split('\n')
            elif s.st_mtime > self.stat.st_mtime:
                # If the file has been changed since last we looked
                self.f.seek(self.stat.st_size)
                lines = self.f.read(s.st_size - self.stat.st_size).strip().split('\n')
            # Now, remember the current stats
            self.stat = s

            # Now that we have all our lines, go ahead and try to match the regex to each line
            counts = dict([(k, 0) for k in self.patterns])
            for line in lines:
                for k, r in self.patterns.items():
                    m = r.search(line)
                    if m:
                        try:
                            # Use the last matching group if found
                            counts[k] += int(m.groups()[-1])
                        except ValueError:
                            logger.warn('Could not parse int from %s. Using 1' % m.gorups()[-1])
                            counts[k] += 1
                        except IndexError:
                            logger.info('No groups in regular expression. Using 1')
                            counts[k] += 1
            return {
                'results' : dict([(k, (v, 'Count')) for k, v in counts.items()])
            }
        except Exception as e:
            raise MetricException(e)
<commit_msg>Fixed a bug when initalizing LogMetric class.<commit_after>#! /usr/bin/env python
#
# Copyright (c) 2011 SEOmoz
#
# Permission is hereby granted, free of charge, to any person obtaining
# a copy of this software and associated documentation files (the
# "Software"), to deal in the Software without restriction, including
# without limitation the rights to use, copy, modify, merge, publish,
# distribute, sublicense, and/or sell copies of the Software, and to
# permit persons to whom the Software is furnished to do so, subject to
# the following conditions:
#
# The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

import re
import os
from sauron import logger
from sauron.metrics import Metric, MetricException

class LogMetric(Metric):
    def __init__(self, name, path, **kwargs):
        Metric.__init__(self, name, **kwargs)
        self.reconfig(name, path, **kwargs)

    def reconfig(self, name, path, **kwargs):
        Metric.reconfig(self, name, **kwargs)
        self.name = name
        self.patterns = dict([(k, re.compile(v)) for k,v in kwargs.items()])
        self.path = path
        try:
            self.f = file(self.path)    # The file object we'll read from
        except IOError as e:
            raise MetricException(e)
        self.stat = os.lstat(self.path) # The stats on that particular file

    def __del__(self):
        try:
            self.f.close()
        except ValueError:
            pass
        except AttributeError:
            pass

    def values(self):
        try:
            # Alright, first get new stats on the file
            s = os.lstat(self.path)
            # The lines we've read
            lines = []
            # Now, see if the file was nuked
            # I'm not sure how this works. Checking inode might not really capture
            # what we're talking about. It certainly happens when the file is replaced,
            # but there /may/ be other times when it changes
            if s.st_ino != self.stat.st_ino:
                logger.warn('Inode for %s has changed' % self.path)
                self.f.close()
                self.f = file(self.path)
                lines = self.f.read(s.st_size).strip().split('\n')
            elif s.st_size < self.stat.st_size:
                logger.warn('File %s has shrunk since last read! Reading from beginning...' % self.path)
                self.f.seek(0)
                lines = self.f.read(s.st_size).strip().split('\n')
            elif s.st_mtime > self.stat.st_mtime:
                # If the file has been changed since last we looked
                self.f.seek(self.stat.st_size)
                lines = self.f.read(s.st_size - self.stat.st_size).strip().split('\n')
            # Now, remember the current stats
            self.stat = s

            # Now that we have all our lines, go ahead and try to match the regex to each line
            counts = dict([(k, 0) for k in self.patterns])
            for line in lines:
                for k, r in self.patterns.items():
                    m = r.search(line)
                    if m:
                        try:
                            # Use the last matching group if found
                            counts[k] += int(m.groups()[-1])
                        except ValueError:
                            logger.warn('Could not parse int from %s. Using 1' % m.gorups()[-1])
                            counts[k] += 1
                        except IndexError:
                            logger.info('No groups in regular expression. Using 1')
                            counts[k] += 1
            return {
                'results' : dict([(k, (v, 'Count')) for k, v in counts.items()])
            }
        except Exception as e:
            raise MetricException(e)
<|endoftext|>


<commit_before>"""
This will parse the large ucd xml into a simple list that is workable and
is fair enough for download and embeding.

starts with the # unicode description/version

format
<unicode> \t <unicode name>

download from
http://www.unicode.org/Public/8.0.0/ucdxml/ucd.all.flat.zip
"""

import os
from xml.etree import cElementTree as ET

# after download it will be in the download folder :)
path = os.path.expanduser("~/Downloads/ucd.all.flat.xml")

tree = ET.parse(path)

flat = []
for i in tree.iter():
    if i.tag.endswith("description"):
        flat.insert(0, "# %s" % i.text)
    if i.tag.endswith("char"):
        n = i.attrib.get("na")
        if n:
            flat.append("%s\t%s" % (i.attrib.get("cp"), n))

f = open("flatUnicode.txt", "w")
f.write("\n".join(flat))
f.close()
<commit_msg>buildFlatUnicodeList: download or specify which ucd.all.flat.xml to use<commit_after>from __future__ import print_function
import os
import sys
import tempfile
from zipfile import ZipFile
from xml.etree import cElementTree as ET
from argparse import ArgumentParser, RawDescriptionHelpFormatter
try:
    from urllib2 import urlopen
except ImportError:
    from urllib.request import urlopen

__doc__ = """
This will parse the large ucd xml into a simple list that is workable and
is fair enough for download and embeding.

starts with the # unicode description/version

format
<unicode> \t <unicode name>
"""

URL = "http://www.unicode.org/Public/{version}/ucdxml/ucd.all.flat.zip"
UNICODE_VERSION = "9.0.0"
UCD_ZIP_FILE = "ucd.all.flat.zip"
UCD_FILE = UCD_ZIP_FILE[:-3] + "xml"
FLAT_FILE = "flatUnicode.txt"

parser = ArgumentParser(description=__doc__,
                        formatter_class=RawDescriptionHelpFormatter)
group = parser.add_mutually_exclusive_group(required=True)
group.add_argument("-u", "--unicode-version",
                   help="Unicode version to use for download and processing")
group.add_argument("-i", "--input", metavar="UCD_FILE",
                   help="Path to copy of {ucd_file} for processing. "
                   "Use if you already have an up-to-date local copy of "
                   "{ucd_file}.".format(ucd_file=UCD_FILE))
options = parser.parse_args()

if options.input:
    path = options.input
else:
    tempdir = tempfile.mkdtemp()
    filename = os.path.join(tempdir, UCD_ZIP_FILE)
    print(">> Downloading {} to {}".format(UCD_ZIP_FILE, filename))
    if options.unicode_version:
        version = options.unicode_version
    else:
        version = UNICODE_VERSION
    url = urlopen(URL.format(version=version))
    with open(filename, "wb") as fp:
        blocksize = 8192
        while True:
            buffer = url.read(blocksize)
            if not buffer:
                break
            fp.write(buffer)
    zipped = ZipFile(filename)
    print(">> Extracting {} to {}"
          .format(UCD_FILE, os.path.join(tempdir, UCD_FILE)))
    path = zipped.extract(UCD_FILE, tempdir)

print(">> Building {} from {}".format(FLAT_FILE, path))
tree = ET.parse(path)

flat = []
for i in tree.iter():
    if i.tag.endswith("description"):
        flat.insert(0, "# %s" % i.text)
    if i.tag.endswith("char"):
        n = i.attrib.get("na")
        if n:
            flat.append("%s\t%s" % (i.attrib.get("cp"), n))

f = open(FLAT_FILE, "w")
f.write("\n".join(flat))
f.close()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

from urllib import urlencode
from django import template
from django.conf import settings

from vault.models import GroupProjects
from identity.keystone import KeystoneNoRequest

import requests
import json

register = template.Library()

@register.simple_tag(takes_context=True)
def with_sidebar(context):
    req = context.get("request")
    body_cls = ""
    if req.COOKIES.get("show_sidebar"):
        body_cls = "with-sidebar "

    return body_cls

@register.simple_tag(takes_context=True)
def get_vault_env(context):
    request = context.get("request")
    envs = ["dev", "qa", "qa1", "qa2", "prod"]

    if settings.ENVIRON in envs:
        return settings.ENVIRON

    if settings.ENVIRON is None and "localhost" in request.get_host():
        return "local"

    return ""

@register.inclusion_tag('vault/set_project.html', takes_context=True)
def set_project(context):
    user = context.get('user')
    groups = user.groups.all()
    request = context.get('request')
    keystone = KeystoneNoRequest()
    group_projects = []

    for group in groups:
        gps = GroupProjects.objects.filter(group=group.id)
        gps_ks = filter(lambda x: x.enabled, keystone.project_list())
        gp_projs = []

        for gp in gps:
            for gp_ks in gps_ks:
                if gp.project == gp_ks.id:
                    gp_projs.append(gp_ks)
                    break

        gp_projs.sort(key=lambda x: x.name.lower())
        group_projects.append({
            'team': group.name,
            'projects': gp_projs
        })

    current_project = {'id': context.get('project_id'),
                       'name': context.get('project_name')}

    if current_project.get('id') is None:
        req = context.get('request')
        current_project['id'] = req.session.get('project_id')
        current_project['name'] = req.session.get('project_name')

    return {
        'current_project': current_project,
        'group_projects': group_projects,
        'has_group': user.groups.count() > 0
    }

@register.simple_tag(takes_context=True)
def get_logout_url(context):
    request = context.get('request')
    logout_url = settings.LOGOUT_URL.format(
        request.META['HTTP_HOST']
    )
    return logout_url

@register.assignment_tag(takes_context=True)
def can_view_team_users_ogs(context):
    user = context.get('user')
    add_og = False
    change_og = False
    delete_og = False

    if len(user.groups.all()) == 0:
        return (add_og and change_og and delete_og)

    group = user.groups.all()[0]

    for permission in group.permissions.all():
        try:
            codename = permission.codename
            exec(codename + ' = True')
        except Exception:
            pass

    return (add_og and change_og and delete_og)

@register.simple_tag()
def url_replace(get_parameters, **kwargs):
    query = get_parameters
    query.update(kwargs)
    return urlencode(query)
<commit_msg>Add "beta" to vault environments<commit_after># -*- coding: utf-8 -*-

from urllib import urlencode
from django import template
from django.conf import settings

from vault.models import GroupProjects
from identity.keystone import KeystoneNoRequest

import requests
import json

register = template.Library()

@register.simple_tag(takes_context=True)
def with_sidebar(context):
    req = context.get("request")
    body_cls = ""
    if req.COOKIES.get("show_sidebar"):
        body_cls = "with-sidebar "

    return body_cls

@register.simple_tag(takes_context=True)
def get_vault_env(context):
    request = context.get("request")
    envs = ["dev", "qa", "qa1", "qa2", "prod", "beta"]

    if settings.ENVIRON in envs:
        return settings.ENVIRON

    if settings.ENVIRON is None and "localhost" in request.get_host():
        return "local"

    return ""

@register.inclusion_tag('vault/set_project.html', takes_context=True)
def set_project(context):
    user = context.get('user')
    groups = user.groups.all()
    request = context.get('request')
    keystone = KeystoneNoRequest()
    group_projects = []

    for group in groups:
        gps = GroupProjects.objects.filter(group=group.id)
        gps_ks = filter(lambda x: x.enabled, keystone.project_list())
        gp_projs = []

        for gp in gps:
            for gp_ks in gps_ks:
                if gp.project == gp_ks.id:
                    gp_projs.append(gp_ks)
                    break

        gp_projs.sort(key=lambda x: x.name.lower())
        group_projects.append({
            'team': group.name,
            'projects': gp_projs
        })

    current_project = {'id': context.get('project_id'),
                       'name': context.get('project_name')}

    if current_project.get('id') is None:
        req = context.get('request')
        current_project['id'] = req.session.get('project_id')
        current_project['name'] = req.session.get('project_name')

    return {
        'current_project': current_project,
        'group_projects': group_projects,
        'has_group': user.groups.count() > 0
    }

@register.simple_tag(takes_context=True)
def get_logout_url(context):
    request = context.get('request')
    logout_url = settings.LOGOUT_URL.format(
        request.META['HTTP_HOST']
    )
    return logout_url

@register.assignment_tag(takes_context=True)
def can_view_team_users_ogs(context):
    user = context.get('user')
    add_og = False
    change_og = False
    delete_og = False

    if len(user.groups.all()) == 0:
        return (add_og and change_og and delete_og)

    group = user.groups.all()[0]

    for permission in group.permissions.all():
        try:
            codename = permission.codename
            exec(codename + ' = True')
        except Exception:
            pass

    return (add_og and change_og and delete_og)

@register.simple_tag()
def url_replace(get_parameters, **kwargs):
    query = get_parameters
    query.update(kwargs)
    return urlencode(query)
<|endoftext|>


<commit_before># Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
# MIT License. See license.txt
from __future__ import unicode_literals

import frappe, json
from frappe.model.meta import is_single
from frappe.modules import load_doctype_module
import frappe.desk.form.meta
import frappe.desk.form.load

@frappe.whitelist()
def get_linked_docs(doctype, name, linkinfo=None, for_doctype=None):
	if isinstance(linkinfo, basestring):
		# additional fields are added in linkinfo
		linkinfo = json.loads(linkinfo)

	results = {}

	if not linkinfo:
		return results

	if for_doctype:
		links = frappe.get_doc(doctype, name).get_link_filters(for_doctype)

		if links:
			linkinfo = links

		if for_doctype in linkinfo:
			# only get linked with for this particular doctype
			linkinfo = { for_doctype: linkinfo.get(for_doctype) }
		else:
			return results

	me = frappe.db.get_value(doctype, name, ["parenttype", "parent"], as_dict=True)

	for dt, link in linkinfo.items():
		link["doctype"] = dt
		link_meta_bundle = frappe.desk.form.load.get_meta_bundle(dt)
		linkmeta = link_meta_bundle[0]
		if not linkmeta.get("issingle"):
			fields = [d.fieldname for d in linkmeta.get("fields", {"in_list_view":1,
				"fieldtype": ["not in", ["Image", "HTML", "Button", "Table"]]})] \
				+ ["name", "modified", "docstatus"]

			if link.get("add_fields"):
				fields += link["add_fields"]

			fields = ["`tab{dt}`.`{fn}`".format(dt=dt, fn=sf.strip()) for sf in fields if sf
				and "`tab" not in sf]

			try:
				if link.get("filters"):
					ret = frappe.get_list(doctype=dt, fields=fields, filters=link.get("filters"))

				elif link.get("get_parent"):
					if me and me.parent and me.parenttype == dt:
						ret = frappe.get_list(doctype=dt, fields=fields,
							filters=[[dt, "name", '=', me.parent]])
					else:
						ret = None

				elif link.get("child_doctype"):
					filters = [[link.get('child_doctype'), link.get("fieldname"), '=', name]]

					# dynamic link
					if link.get("doctype_fieldname"):
						filters.append([link.get('child_doctype'), link.get("doctype_fieldname"), "=", doctype])

					ret = frappe.get_list(doctype=dt, fields=fields, filters=filters)

				else:
					if link.get("fieldname"):
						filters = [[dt, link.get("fieldname"), '=', name]]
						# dynamic link
						if link.get("doctype_fieldname"):
							filters.append([dt, link.get("doctype_fieldname"), "=", doctype])
						ret = frappe.get_list(doctype=dt, fields=fields, filters=filters)

					else:
						ret = None

			except frappe.PermissionError:
				if frappe.local.message_log:
					frappe.local.message_log.pop()

				continue

			if ret:
				results[dt] = ret

	return results

@frappe.whitelist()
def get_linked_doctypes(doctype):
	"""add list of doctypes this doctype is 'linked' with.

	Example, for Customer:

		{"Address": {"fieldname": "customer"}..}
	"""
	return frappe.cache().hget("linked_doctypes", doctype, lambda: _get_linked_doctypes(doctype))

def _get_linked_doctypes(doctype):
	ret = {}

	# find fields where this doctype is linked
	ret.update(get_linked_fields(doctype))

	ret.update(get_dynamic_linked_fields(doctype))

	# find links of parents
	links = frappe.db.sql("""select dt from `tabCustom Field`
		where (fieldtype="Table" and options=%s)""", (doctype))
	links += frappe.db.sql("""select parent from tabDocField
		where (fieldtype="Table" and options=%s)""", (doctype))

	for dt, in links:
		if not dt in ret:
			ret[dt] = {"get_parent": True}

	for dt in ret.keys():
		try:
			doctype_module = load_doctype_module(dt)
		except ImportError:
			# in case of Custom DocType
			continue

		if getattr(doctype_module, "exclude_from_linked_with", False):
			del ret[dt]

	return ret

def get_linked_fields(doctype):
	links = frappe.db.sql("""select parent, fieldname from tabDocField
		where (fieldtype="Link" and options=%s)
		or (fieldtype="Select" and options=%s)""", (doctype, "link:"+ doctype))
	links += frappe.db.sql("""select dt as parent, fieldname from `tabCustom Field`
		where (fieldtype="Link" and options=%s)
		or (fieldtype="Select" and options=%s)""", (doctype, "link:"+ doctype))

	links = dict(links)

	ret = {}

	if links:
		for dt in links:
			ret[dt] = { "fieldname": links[dt] }

		# find out if linked in a child table
		for parent, options in frappe.db.sql("""select parent, options from tabDocField
			where fieldtype="Table"
				and options in (select name from tabDocType
					where istable=1 and name in (%s))""" % ", ".join(["%s"] * len(links)) ,tuple(links)):

			ret[parent] = {"child_doctype": options, "fieldname": links[options] }
			if options in ret:
				del ret[options]

	return ret

def get_dynamic_linked_fields(doctype):
	ret = {}

	links = frappe.db.sql("""select parent as doctype, fieldname, options as doctype_fieldname
		from `tabDocField` where fieldtype='Dynamic Link'""", as_dict=True)
	links += frappe.db.sql("""select dt as doctype, fieldname, options as doctype_fieldname
		from `tabCustom Field` where fieldtype='Dynamic Link'""", as_dict=True)

	for df in links:
		if is_single(df.doctype):
			continue

		# optimized to get both link exists and parenttype
		possible_link = frappe.db.sql("""select distinct `{doctype_fieldname}`, parenttype
			from `tab{doctype}` where `{doctype_fieldname}`=%s""".format(**df), doctype, as_dict=True)
		if possible_link:
			for d in possible_link:
				# is child
				if d.parenttype:
					ret[d.parenttype] = {
						"child_doctype": df.doctype,
						"fieldname": df.fieldname,
						"doctype_fieldname": df.doctype_fieldname
					}

				else:
					ret[df.doctype] = {
						"fieldname": df.fieldname,
						"doctype_fieldname": df.doctype_fieldname
					}

	return ret<commit_msg>[fix] When user clicks on link in invoice there are duplicate DNs are showing against it (#3204)<commit_after># Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
# MIT License. See license.txt
from __future__ import unicode_literals

import frappe, json
from frappe.model.meta import is_single
from frappe.modules import load_doctype_module
import frappe.desk.form.meta
import frappe.desk.form.load

@frappe.whitelist()
def get_linked_docs(doctype, name, linkinfo=None, for_doctype=None):
	if isinstance(linkinfo, basestring):
		# additional fields are added in linkinfo
		linkinfo = json.loads(linkinfo)

	results = {}

	if not linkinfo:
		return results

	if for_doctype:
		links = frappe.get_doc(doctype, name).get_link_filters(for_doctype)

		if links:
			linkinfo = links

		if for_doctype in linkinfo:
			# only get linked with for this particular doctype
			linkinfo = { for_doctype: linkinfo.get(for_doctype) }
		else:
			return results

	me = frappe.db.get_value(doctype, name, ["parenttype", "parent"], as_dict=True)

	for dt, link in linkinfo.items():
		link["doctype"] = dt
		link_meta_bundle = frappe.desk.form.load.get_meta_bundle(dt)
		linkmeta = link_meta_bundle[0]
		if not linkmeta.get("issingle"):
			fields = [d.fieldname for d in linkmeta.get("fields", {"in_list_view":1,
				"fieldtype": ["not in", ["Image", "HTML", "Button", "Table"]]})] \
				+ ["name", "modified", "docstatus"]

			if link.get("add_fields"):
				fields += link["add_fields"]

			fields = ["`tab{dt}`.`{fn}`".format(dt=dt, fn=sf.strip()) for sf in fields if sf
				and "`tab" not in sf]

			try:
				if link.get("filters"):
					ret = frappe.get_list(doctype=dt, fields=fields, filters=link.get("filters"))

				elif link.get("get_parent"):
					if me and me.parent and me.parenttype == dt:
						ret = frappe.get_list(doctype=dt, fields=fields,
							filters=[[dt, "name", '=', me.parent]])
					else:
						ret = None

				elif link.get("child_doctype"):
					filters = [[link.get('child_doctype'), link.get("fieldname"), '=', name]]

					# dynamic link
					if link.get("doctype_fieldname"):
						filters.append([link.get('child_doctype'), link.get("doctype_fieldname"), "=", doctype])

					ret = frappe.get_list(doctype=dt, fields=fields, filters=filters, distinct=True)

				else:
					if link.get("fieldname"):
						filters = [[dt, link.get("fieldname"), '=', name]]
						# dynamic link
						if link.get("doctype_fieldname"):
							filters.append([dt, link.get("doctype_fieldname"), "=", doctype])
						ret = frappe.get_list(doctype=dt, fields=fields, filters=filters)

					else:
						ret = None

			except frappe.PermissionError:
				if frappe.local.message_log:
					frappe.local.message_log.pop()

				continue

			if ret:
				results[dt] = ret

	return results

@frappe.whitelist()
def get_linked_doctypes(doctype):
	"""add list of doctypes this doctype is 'linked' with.

	Example, for Customer:

		{"Address": {"fieldname": "customer"}..}
	"""
	return frappe.cache().hget("linked_doctypes", doctype, lambda: _get_linked_doctypes(doctype))

def _get_linked_doctypes(doctype):
	ret = {}

	# find fields where this doctype is linked
	ret.update(get_linked_fields(doctype))

	ret.update(get_dynamic_linked_fields(doctype))

	# find links of parents
	links = frappe.db.sql("""select dt from `tabCustom Field`
		where (fieldtype="Table" and options=%s)""", (doctype))
	links += frappe.db.sql("""select parent from tabDocField
		where (fieldtype="Table" and options=%s)""", (doctype))

	for dt, in links:
		if not dt in ret:
			ret[dt] = {"get_parent": True}

	for dt in ret.keys():
		try:
			doctype_module = load_doctype_module(dt)
		except ImportError:
			# in case of Custom DocType
			continue

		if getattr(doctype_module, "exclude_from_linked_with", False):
			del ret[dt]

	return ret

def get_linked_fields(doctype):
	links = frappe.db.sql("""select parent, fieldname from tabDocField
		where (fieldtype="Link" and options=%s)
		or (fieldtype="Select" and options=%s)""", (doctype, "link:"+ doctype))
	links += frappe.db.sql("""select dt as parent, fieldname from `tabCustom Field`
		where (fieldtype="Link" and options=%s)
		or (fieldtype="Select" and options=%s)""", (doctype, "link:"+ doctype))

	links = dict(links)

	ret = {}

	if links:
		for dt in links:
			ret[dt] = { "fieldname": links[dt] }

		# find out if linked in a child table
		for parent, options in frappe.db.sql("""select parent, options from tabDocField
			where fieldtype="Table"
				and options in (select name from tabDocType
					where istable=1 and name in (%s))""" % ", ".join(["%s"] * len(links)) ,tuple(links)):

			ret[parent] = {"child_doctype": options, "fieldname": links[options] }
			if options in ret:
				del ret[options]

	return ret

def get_dynamic_linked_fields(doctype):
	ret = {}

	links = frappe.db.sql("""select parent as doctype, fieldname, options as doctype_fieldname
		from `tabDocField` where fieldtype='Dynamic Link'""", as_dict=True)
	links += frappe.db.sql("""select dt as doctype, fieldname, options as doctype_fieldname
		from `tabCustom Field` where fieldtype='Dynamic Link'""", as_dict=True)

	for df in links:
		if is_single(df.doctype):
			continue

		# optimized to get both link exists and parenttype
		possible_link = frappe.db.sql("""select distinct `{doctype_fieldname}`, parenttype
			from `tab{doctype}` where `{doctype_fieldname}`=%s""".format(**df), doctype, as_dict=True)
		if possible_link:
			for d in possible_link:
				# is child
				if d.parenttype:
					ret[d.parenttype] = {
						"child_doctype": df.doctype,
						"fieldname": df.fieldname,
						"doctype_fieldname": df.doctype_fieldname
					}

				else:
					ret[df.doctype] = {
						"fieldname": df.fieldname,
						"doctype_fieldname": df.doctype_fieldname
					}

	return ret<|endoftext|>


<commit_before># Copyright 2021 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     https://www.apache.org/licenses/LICENSE-2.0
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Implements Evaluation Metrics."""

from typing import Any, Dict, List
import numpy as np
from t5.evaluation import metrics

def measure_multiple_choice_grade(samples):
    """Scoring based on argmax of `log_prob`.

    Args:
      samples: List of dictionaries with 'target_scores' field.

    Returns:
      Average score on task.
    """
    count = 0

    def argmax(array):
        """argmax with deterministic pseudorandom tie breaking."""
        max_indices = np.arange(len(array))[array == np.max(array)]
        idx = hash(np.asarray(array).tobytes()) % len(max_indices)
        return max_indices[idx]

    for sample in samples:
        choice = sample["choice"][argmax(sample["log_prob"])]
        count += sample["target_scores"][choice]

    return count / len(samples)

def exact_str_match_fn(targets: List[str], responses: List[str]) -> Dict[str, Any]:
    """Exact match between targets and responses."""
    # aitor: might want to use these metrics after removing irrelevant words.
    # they do something like this in see t5.data.postprocessors.wsc_simple
    count = 0

    for target, response in zip(targets, responses):
        if not isinstance(target, list):
            target = [target]
        if response in target:
            count += 1

    return {"exact_str_match": count / len(targets)}

MULTIPLE_CHOICE_FN = {"multiple_choice_grade": measure_multiple_choice_grade}

def measure_multiple_choice_metrics(
    samples: List[Dict[str, Any]], multiple_choice_metrics: List[str]
) -> Dict[str, float]:
    """Measures multiple choice metrics from samples."""

    results = {}
    for metric in multiple_choice_metrics:
        results[metric] = MULTIPLE_CHOICE_FN[metric](samples)

    return results

def rouge_fn(targets: List[List[str]], responses: List[str]
) -> Dict[str, float]:
    """Computes ROUGE by taking the max ROUGE-N per reference and N."""
    # Following strategy from https://www.aclweb.org/anthology/W04-1013/.
    # Identify best reference per response and ROUGE type.
    rouge_types = ["rouge1", "rouge2", "rougeLsum"]
    max_references = {rouge_type: [] for rouge_type in rouge_types}
    for targ_for_resp, resp in zip(targets, responses):
        # Compute individual scores per example/ref pair.
        resp_scores = [metrics.rouge([t], [resp]) for t in targ_for_resp]
        # Find best scoring references for generated output and ROUGE type.
        for rouge_type in rouge_types:
            best_score_index = max(
                range(len(resp_scores)), key=lambda x: resp_scores[x][rouge_type]
            )
            best_ref = targ_for_resp[best_score_index]
            # Add the reference to the new reference list.
            max_references[rouge_type].append(best_ref)
    # Compute metric for each of the reference lists for a ref type.
    results = {}
    for rouge_type in rouge_types:
        results[rouge_type] = metrics.rouge(max_references[rouge_type], responses)[rouge_type]
    return results

def bleu_fn(targets: List[List[str]], responses: List[str]
) -> Dict[str, float]:
    max_refs = max([len(target) for target in targets])
    if len(list(set([len(target) for target in targets]))) > 1:
        # Padding needed for sacrebleu.
        for target in targets:
            if len(target) < max_refs:
                target += [target[0]] * (max_refs - len(target))

    new_targets = [[target[i] for target in targets] for i in range(max_refs)]
    return metrics.bleu(new_targets, responses)

def sequence_f1_fn(targets: List[List[str]], responses: List[List[str]]
) -> Dict[str, float]:
    f1s = []
    for (t, p) in zip(targets, responses):
        f1s.append(metrics.sequence_accuracy(t, p)['sequence_accuracy'])
    return {"f1": np.mean(np.array(f1s))}

GENERATIVE_FN = {
    "bleu": bleu_fn,
    "rouge": rouge_fn,
    "exact_str_match": exact_str_match_fn,
    "sequence_f1": sequence_f1_fn
}

def measure_generative_metrics(
    samples: List[Dict[str, Any]], generative_metrics: List[str]
) -> Dict[str, float]:
    """Measure of generative metrics."""

    results = {}

    if not generative_metrics:
        return results

    responses = [sample["generative_response"] for sample in samples]
    targets = [sample["target"] for sample in samples]

    for metric in generative_metrics:
        if metric == 'sequence_f1':
            # TODO way to properly cast responses to list of predictions
            responses = []
            for target in targets:
                responses.append([samples[0]["generative_response"]]*len(target))
        result = GENERATIVE_FN[metric](targets, responses)
        results.update(result)

    return results
<commit_msg>task_metrics redone<commit_after># Copyright 2021 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     https://www.apache.org/licenses/LICENSE-2.0
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Implements Evaluation Metrics."""

from typing import Any, Dict, List
import numpy as np
from t5.evaluation import metrics

def measure_multiple_choice_grade(samples):
    """Scoring based on argmax of `log_prob`.

    Args:
      samples: List of dictionaries with 'target_scores' field.

    Returns:
      Average score on task.
    """
    count = 0

    def argmax(array):
        """argmax with deterministic pseudorandom tie breaking."""
        max_indices = np.arange(len(array))[array == np.max(array)]
        idx = hash(np.asarray(array).tobytes()) % len(max_indices)
        return max_indices[idx]

    for sample in samples:
        choice = sample["choice"][argmax(sample["log_prob"])]
        count += sample["target_scores"][choice]

    return count / len(samples)

def exact_str_match_fn(targets: List[str], responses: List[str]) -> Dict[str, Any]:
    """Exact match between targets and responses."""
    # aitor: might want to use these metrics after removing irrelevant words.
    # they do something like this in see t5.data.postprocessors.wsc_simple
    count = 0

    for target, response in zip(targets, responses):
        if not isinstance(target, list):
            target = [target]
        if response in target:
            count += 1

    return {"exact_str_match": count / len(targets)}

MULTIPLE_CHOICE_FN = {"multiple_choice_grade": measure_multiple_choice_grade}

def measure_multiple_choice_metrics(
    samples: List[Dict[str, Any]], multiple_choice_metrics: List[str]
) -> Dict[str, float]:
    """Measures multiple choice metrics from samples."""

    results = {}
    for metric in multiple_choice_metrics:
        results[metric] = MULTIPLE_CHOICE_FN[metric](samples)

    return results

def rouge_fn(targets: List[List[str]], responses: List[str]
) -> Dict[str, float]:
    """Computes ROUGE by taking the max ROUGE-N per reference and N."""
    # Following strategy from https://www.aclweb.org/anthology/W04-1013/.
    # Identify best reference per response and ROUGE type.
    rouge_types = ["rouge1", "rouge2", "rougeLsum"]
    max_references = {rouge_type: [] for rouge_type in rouge_types}
    for targ_for_resp, resp in zip(targets, responses):
        # Compute individual scores per example/ref pair.
        resp_scores = [metrics.rouge([t], [resp]) for t in targ_for_resp]
        # Find best scoring references for generated output and ROUGE type.
        for rouge_type in rouge_types:
            best_score_index = max(
                range(len(resp_scores)), key=lambda x: resp_scores[x][rouge_type]
            )
            best_ref = targ_for_resp[best_score_index]
            # Add the reference to the new reference list.
            max_references[rouge_type].append(best_ref)
    # Compute metric for each of the reference lists for a ref type.
    results = {}
    for rouge_type in rouge_types:
        results[rouge_type] = metrics.rouge(max_references[rouge_type], responses)[rouge_type]
    return results

def bleu_fn(targets: List[List[str]], responses: List[str]
) -> Dict[str, float]:
    max_refs = max([len(target) for target in targets])
    if len(list(set([len(target) for target in targets]))) > 1:
        # Padding needed for sacrebleu.
        for target in targets:
            if len(target) < max_refs:
                target += [target[0]] * (max_refs - len(target))

    new_targets = [[target[i] for target in targets] for i in range(max_refs)]
    return metrics.bleu(new_targets, responses)

def sequence_f1_fn(targets: List[List[str]], responses: List[str]
) -> Dict[str, float]:
    f1s = []
    for (t, p) in zip(targets, responses):
        # pad or trim to size of target
        predicted_labels = p.split(' ')
        target_labels = t[0].split(' ')
        if len(predicted_labels) > len(target_labels):
            predicted_labels = predicted_labels[:len(target_labels)]
        elif len(predicted_labels) < len(target_labels):
            predicted_labels.extend(['O']*(len(target_labels)-len(predicted_labels)))
        f1s.append(metrics.sequence_accuracy(target_labels, predicted_labels)['sequence_accuracy'])
    return {"f1": np.mean(np.array(f1s))}

GENERATIVE_FN = {
    "bleu": bleu_fn,
    "rouge": rouge_fn,
    "exact_str_match": exact_str_match_fn,
    "sequence_f1": sequence_f1_fn
}

def measure_generative_metrics(
    samples: List[Dict[str, Any]], generative_metrics: List[str]
) -> Dict[str, float]:
    """Measure of generative metrics."""

    results = {}

    if not generative_metrics:
        return results

    responses = [sample["generative_response"] for sample in samples]
    targets = [sample["target"] for sample in samples]

    for metric in generative_metrics:
        result = GENERATIVE_FN[metric](targets, responses)
        results.update(result)

    return results
<|endoftext|>


<commit_before># This file is part of Indico.
# Copyright (C) 2002 - 2015 European Organization for Nuclear Research (CERN).
#
# Indico is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License as
# published by the Free Software Foundation; either version 3 of the
# License, or (at your option) any later version.
#
# Indico is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Indico; if not, see <http://www.gnu.org/licenses/>.

from __future__ import unicode_literals

from indico.util.user import retrieve_principals, principal_to_tuple
from indico.core.config import Config

from MaKaC.authentication.AuthenticationMgr import AuthenticatorMgr

def get_auth_users():
    """Returns a list of authorized users

    :return: list of Avatar/Group objects
    """
    from indico_vc_vidyo.plugin import VidyoPlugin
    return retrieve_principals(VidyoPlugin.settings.get('authorized_users'))

def is_auth_user(user):
    """Checks if a user is authorized"""
    return any(principal.containsUser(user) for principal in get_auth_users())

def iter_user_identities(avatar):
    """Iterates over all existing user identities that can be used with Vidyo"""
    from indico_vc_vidyo.plugin import VidyoPlugin

    authenticators = (a.strip() for a in VidyoPlugin.settings.get('authenticators').split(','))
    return (identity.getLogin()
            for auth in authenticators
            for identity in avatar.getIdentityByAuthenticatorId(auth))

def get_avatar_from_identity(settings, identity):
    """Get an actual avatar object from an auth identity"""
    authenticators = list(auth.strip() for auth in settings.get('authenticators').split(','))
    return next((avatar for auth_id, avatar in AuthenticatorMgr().getAvatarByLogin(identity, authenticators).iteritems()
                if avatar is not None), None)

def iter_extensions(prefix, event_id):
    """Return extension (prefix + event_id) with an optional suffix which is
       incremented step by step in case of collision
    """
    extension = '{prefix}{event_id}'.format(prefix=prefix, event_id=event_id,)
    yield extension
    suffix = 1
    while True:
        yield '{extension}{suffix}'.format(extension=extension, suffix=suffix)
        suffix += 1

def update_room_from_obj(settings, vc_room, room_obj):
    """Updates a VCRoom DB object using a SOAP room object returned by the API"""
    config = Config.getInstance()

    vc_room.name = room_obj.name

    if room_obj.ownerName != vc_room.data['owner_identity']:
        avatar = get_avatar_from_identity(settings, room_obj.ownerName)
        # if the owner does not exist any more (e.g. was changed on the server),
        # use the janitor user as a placeholder
        vc_room.data['owner'] = (('Avatar', config.getJanitorUserId()) if avatar is None
                                 else principal_to_tuple(avatar))

    vc_room.data.update({
        'description': room_obj.description,
        'vidyo_id': unicode(room_obj.roomID),
        'url': room_obj.RoomMode.roomURL,
        'owner_identity': room_obj.ownerName,
        'room_pin': room_obj.RoomMode.roomPIN if room_obj.RoomMode.hasPIN else "",
        'moderation_pin': room_obj.RoomMode.moderatorPIN if room_obj.RoomMode.hasModeratorPIN else "",
    })
    vc_room.vidyo_extension.value = int(room_obj.extension)
<commit_msg>VC/Vidyo: Remove extra comma<commit_after># This file is part of Indico.
# Copyright (C) 2002 - 2015 European Organization for Nuclear Research (CERN).
#
# Indico is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License as
# published by the Free Software Foundation; either version 3 of the
# License, or (at your option) any later version.
#
# Indico is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Indico; if not, see <http://www.gnu.org/licenses/>.

from __future__ import unicode_literals

from indico.util.user import retrieve_principals, principal_to_tuple
from indico.core.config import Config

from MaKaC.authentication.AuthenticationMgr import AuthenticatorMgr

def get_auth_users():
    """Returns a list of authorized users

    :return: list of Avatar/Group objects
    """
    from indico_vc_vidyo.plugin import VidyoPlugin
    return retrieve_principals(VidyoPlugin.settings.get('authorized_users'))

def is_auth_user(user):
    """Checks if a user is authorized"""
    return any(principal.containsUser(user) for principal in get_auth_users())

def iter_user_identities(avatar):
    """Iterates over all existing user identities that can be used with Vidyo"""
    from indico_vc_vidyo.plugin import VidyoPlugin

    authenticators = (a.strip() for a in VidyoPlugin.settings.get('authenticators').split(','))
    return (identity.getLogin()
            for auth in authenticators
            for identity in avatar.getIdentityByAuthenticatorId(auth))

def get_avatar_from_identity(settings, identity):
    """Get an actual avatar object from an auth identity"""
    authenticators = list(auth.strip() for auth in settings.get('authenticators').split(','))
    return next((avatar for auth_id, avatar in AuthenticatorMgr().getAvatarByLogin(identity, authenticators).iteritems()
                if avatar is not None), None)

def iter_extensions(prefix, event_id):
    """Return extension (prefix + event_id) with an optional suffix which is
       incremented step by step in case of collision
    """
    extension = '{prefix}{event_id}'.format(prefix=prefix, event_id=event_id)
    yield extension
    suffix = 1
    while True:
        yield '{extension}{suffix}'.format(extension=extension, suffix=suffix)
        suffix += 1

def update_room_from_obj(settings, vc_room, room_obj):
    """Updates a VCRoom DB object using a SOAP room object returned by the API"""
    config = Config.getInstance()

    vc_room.name = room_obj.name

    if room_obj.ownerName != vc_room.data['owner_identity']:
        avatar = get_avatar_from_identity(settings, room_obj.ownerName)
        # if the owner does not exist any more (e.g. was changed on the server),
        # use the janitor user as a placeholder
        vc_room.data['owner'] = (('Avatar', config.getJanitorUserId()) if avatar is None
                                 else principal_to_tuple(avatar))

    vc_room.data.update({
        'description': room_obj.description,
        'vidyo_id': unicode(room_obj.roomID),
        'url': room_obj.RoomMode.roomURL,
        'owner_identity': room_obj.ownerName,
        'room_pin': room_obj.RoomMode.roomPIN if room_obj.RoomMode.hasPIN else "",
        'moderation_pin': room_obj.RoomMode.moderatorPIN if room_obj.RoomMode.hasModeratorPIN else "",
    })
    vc_room.vidyo_extension.value = int(room_obj.extension)
<|endoftext|>