<commit_before>#!/usr/bin/env python
# Copyright (c) 2012 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

""" Upload actual GM results to the skia-autogen SVN repository to aid in
rebaselining. """

from build_step import BuildStep
from utils import merge_into_svn
from utils import misc
import optparse
import os
import sys

# Class we can set attributes on, to emulate optparse-parsed options.
# TODO: Remove the need for this by passing parameters into MergeIntoSvn() some
# other way
class Options(object):
  pass

class UploadGMResults(BuildStep):
  def __init__(self, args, attempts=5):
    super(UploadGMResults, self).__init__(args, attempts)

  def _Run(self, args):
    # TODO these constants should actually be shared by multiple build steps
    gm_actual_basedir = os.path.join(os.pardir, os.pardir, 'gm', 'actual')
    gm_merge_basedir = os.path.join(os.pardir, os.pardir, 'gm', 'merge')
    autogen_svn_baseurl = 'https://skia-autogen.googlecode.com/svn'
    gm_actual_svn_baseurl = '%s/%s' % (autogen_svn_baseurl, 'gm-actual')
    autogen_svn_username_file = args['autogen_svn_username_file']
    autogen_svn_password_file = args['autogen_svn_password_file']

    # Call MergeIntoSvn to actually perform the work.
    # TODO: We should do something a bit more sophisticated, to address
    # https://code.google.com/p/skia/issues/detail?id=720 ('UploadGMs step
    # should be skipped when re-running old revisions of the buildbot')
    merge_options = Options()
    merge_options.commit_message = 'UploadGMResults of r%s on %s' % (
        self._revision, args['builder_name'])
    merge_options.dest_svn_url = '%s/%s/%s/%s' % (
        gm_actual_svn_baseurl, args['gm_image_subdir'], args['builder_name'],
        args['gm_image_subdir'])
    merge_options.merge_dir_path = os.path.join(gm_merge_basedir,
                                                args['gm_image_subdir'])
    merge_options.source_dir_path = os.path.join(gm_actual_basedir,
                                                 args['gm_image_subdir'])
    merge_options.svn_password_file = autogen_svn_password_file
    merge_options.svn_username_file = autogen_svn_username_file
    merge_into_svn.MergeIntoSvn(merge_options)

if '__main__' == __name__:
  sys.exit(BuildStep.Run(UploadGMResults))
<commit_msg>Disable UploadGMResults step since skia-autogen is full<commit_after>#!/usr/bin/env python
# Copyright (c) 2012 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

""" Upload actual GM results to the skia-autogen SVN repository to aid in
rebaselining. """

from build_step import BuildStep
from utils import merge_into_svn
from utils import misc
import optparse
import os
import sys

# Class we can set attributes on, to emulate optparse-parsed options.
# TODO: Remove the need for this by passing parameters into MergeIntoSvn() some
# other way
class Options(object):
  pass

class UploadGMResults(BuildStep):
  def __init__(self, args, attempts=5):
    super(UploadGMResults, self).__init__(args, attempts)

  def _Run(self, args):
    # Step disabled since skia-autogen has reached its quota.  Will re-enable.
    return
    # TODO these constants should actually be shared by multiple build steps
    gm_actual_basedir = os.path.join(os.pardir, os.pardir, 'gm', 'actual')
    gm_merge_basedir = os.path.join(os.pardir, os.pardir, 'gm', 'merge')
    autogen_svn_baseurl = 'https://skia-autogen.googlecode.com/svn'
    gm_actual_svn_baseurl = '%s/%s' % (autogen_svn_baseurl, 'gm-actual')
    autogen_svn_username_file = args['autogen_svn_username_file']
    autogen_svn_password_file = args['autogen_svn_password_file']

    # Call MergeIntoSvn to actually perform the work.
    # TODO: We should do something a bit more sophisticated, to address
    # https://code.google.com/p/skia/issues/detail?id=720 ('UploadGMs step
    # should be skipped when re-running old revisions of the buildbot')
    merge_options = Options()
    merge_options.commit_message = 'UploadGMResults of r%s on %s' % (
        self._revision, args['builder_name'])
    merge_options.dest_svn_url = '%s/%s/%s/%s' % (
        gm_actual_svn_baseurl, args['gm_image_subdir'], args['builder_name'],
        args['gm_image_subdir'])
    merge_options.merge_dir_path = os.path.join(gm_merge_basedir,
                                                args['gm_image_subdir'])
    merge_options.source_dir_path = os.path.join(gm_actual_basedir,
                                                 args['gm_image_subdir'])
    merge_options.svn_password_file = autogen_svn_password_file
    merge_options.svn_username_file = autogen_svn_username_file
    merge_into_svn.MergeIntoSvn(merge_options)

if '__main__' == __name__:
  sys.exit(BuildStep.Run(UploadGMResults))
<|endoftext|>


<commit_before>#!/usr/bin/env python
#########################################################################################
#
# Test function for sct_process_segmentation script
#
#   replace the shell test script in sct 1.0
#
# ---------------------------------------------------------------------------------------
# Copyright (c) 2014 Polytechnique Montreal <www.neuro.polymtl.ca>
# Author: Augustin Roux
# modified: 2014/09/28
#
# About the license: see the file LICENSE.TXT
#########################################################################################
# TODO: make it compatible with isct_test_function
# TODO: add log file

import sct_utils as sct
from pandas import DataFrame
import time

def test(path_data='', parameters=''):

    # initializations
    result_rmse = float('NaN')
    result_dist_max = float('NaN')
    time_start = time.time()

    if not parameters:
        parameters = '-i t2/t2_seg.nii.gz'

    # parameters
    folder_data = 't2/'
    file_data = 't2_seg.nii.gz'

    # define command
    cmd = 'sct_process_segmentation -i ' + path_data + folder_data + file_data \
          + ' -p centerline' \
          + ' -v 1'
    output = '\n====================================================================================================\n'+cmd+'\n====================================================================================================\n\n'  # copy command
    status, o = sct.run(cmd, 0)
    output += o

    # define command
    cmd = 'sct_process_segmentation -i ' + path_data + folder_data + file_data \
          + ' -p length'
    output = '\n====================================================================================================\n'+cmd+'\n====================================================================================================\n\n'  # copy command
    status, o = sct.run(cmd, 0)
    output += o

    # define command
    # cmd = 'sct_process_segmentation -i ' + path_data + folder_data + file_data \
    #       + ' -p csa' \
    #       + ' -size 1'\
    #       + ' -r 0'\
    #       + ' -v 1'
    # output = '\n====================================================================================================\n'+cmd+'\n====================================================================================================\n\n'  # copy command
    # status, o = sct.run(cmd, 0)
    # output += o

    # CSA with integrity testing on various angles of segmentation
    import numpy as np
    import nibabel as nib
    import pickle
    size_data = 31
    size_seg = 2  # segmentation size in the X and Y dimension is size_seg*2+1
    csa_truevalue = 25.0
    th_csa_error = 1  # maximum threshold of CSA error allowed
    # Initialise numpy volumes
    data_seg = np.zeros((size_data, size_data, size_data), dtype=np.int16)
    # create labels=1 in a shape of a parallelipiped: 5x5xnz
    data_seg[np.round(size_data / 2) - size_seg:np.round(size_data / 2) + size_seg + 1,
             np.round(size_data / 2) - size_seg:np.round(size_data / 2) + size_seg + 1,
             0: size_data] = 1
    # save as nifti
    img = nib.Nifti1Image(data_seg, np.eye(4))
    nib.save(img, 'data_seg.nii.gz')
    # create nifti with rotated header
    # build rotation matrix
    from numpy import matrix, cos, sin
    alpha = 0.7854  # corresponds to 45deg angle
    beta = 0.7854  # corresponds to 45deg angle
    gamma = 0.7854  # corresponds to 45deg angle
    rotation_matrix = matrix([[cos(alpha)*cos(beta), cos(alpha)*sin(beta)*sin(gamma)-sin(alpha)*cos(gamma), cos(alpha)*sin(beta)*cos(gamma)+sin(alpha)*sin(gamma)],
                              [sin(alpha)*cos(beta), sin(alpha)*sin(beta)*sin(gamma)+cos(alpha)*cos(gamma), sin(alpha)*sin(beta)*cos(gamma)-cos(alpha)*sin(gamma)],
                              [-sin(beta), cos(beta)*sin(gamma), cos(beta)*cos(gamma)]])
    affine_matrix = np.eye(4)
    affine_matrix[0:3, 0:3] = rotation_matrix
    img_rothd = nib.Nifti1Image(data_seg, affine_matrix)
    nib.save(img_rothd, 'data_seg_rothd.nii.gz')
    # rotate src image
    sct.run('sct_label_utils -i data_seg.nii.gz -create 13,13,0,3:13,13,15,2:13,16,15,4:13,13,30,1 -o data_rot_src.nii.gz', 0)
    sct.run('sct_label_utils -i data_seg.nii.gz -create 4,13,6,3:13,13,15,2:13,16,15,4:22,13,26,1 -o data_rot_dest.nii.gz', 0)
    sct.run('sct_register_multimodal -i data_seg.nii.gz -d data_seg.nii.gz -ilabel data_rot_src.nii.gz -dlabel data_rot_dest.nii.gz -param step=0,type=label,dof=Rx_Ry:step=1,type=im,algo=syn,iter=0 -x linear', 0)
    # compute CSA
    cmd = 'sct_process_segmentation -i data_seg.nii.gz' \
          + ' -p csa' \
          + ' -size 0' \
          + ' -r 0' \
          + ' -v 1' \
          + ' -z 5:23' \
          + ' -ofolder csa'
    output = '\n====================================================================================================\n' + cmd + '\n====================================================================================================\n\n'  # copy command
    status, o = sct.run(cmd, 0)
    output += o
    # check integrity
    csa = pickle.load(open('csa/csa_mean.pickle', 'rb'))
    csa_error = np.abs(csa['MEAN across slices'] - csa_truevalue)
    if csa_error > th_csa_error:
        status = 99
        output += '\nWARNING: CSA_ERROR = ' + str(csa_error) + ' < ' + str(th_csa_error)

    # compute CSA on rotated image
    cmd = 'sct_process_segmentation -i data_seg_reg.nii.gz' \
          + ' -p csa' \
          + ' -size 0' \
          + ' -r 0' \
          + ' -v 1' \
          + ' -z 5:23' \
          + ' -ofolder csa_rot'
    output = '\n====================================================================================================\n' + cmd + '\n====================================================================================================\n\n'  # copy command
    status, o = sct.run(cmd, 0)
    output += o
    # check integrity
    csa = pickle.load(open('csa_rot/csa_mean.pickle', 'rb'))
    csa_error = np.abs(csa['MEAN across slices'] - csa_truevalue)
    if csa_error > th_csa_error:
        status = 99
        output += '\nWARNING: CSA_ERROR = ' + str(csa_error) + ' < ' + str(th_csa_error)

    # compute CSA on rotated header
    cmd = 'sct_process_segmentation -i data_seg_rothd.nii.gz' \
          + ' -p csa' \
          + ' -size 0' \
          + ' -r 0' \
          + ' -v 1' \
          + ' -z 5:23' \
          + ' -ofolder csa_rothd'
    output = '\n====================================================================================================\n' + cmd + '\n====================================================================================================\n\n'  # copy command
    status, o = sct.run(cmd, 0)
    output += o
    # check integrity
    csa = pickle.load(open('csa_rothd/csa_mean.pickle', 'rb'))
    csa_error = np.abs(csa['MEAN across slices'] - csa_truevalue)
    if csa_error > th_csa_error:
        status = 99
        output += '\nWARNING: CSA_ERROR = ' + str(csa_error) + ' < ' + str(th_csa_error)

    # transform results into Pandas structure
    duration = time.time() - time_start
    results = DataFrame(data={'status': int(status), 'output': output, 'csa_error': csa_error, 'duration': duration}, index=[path_data])
    return status, output, results

if __name__ == "__main__":
    # call main function
    test()
<commit_msg>TEST: changed test angle for process segmentation<commit_after>#!/usr/bin/env python
#########################################################################################
#
# Test function for sct_process_segmentation script
#
#   replace the shell test script in sct 1.0
#
# ---------------------------------------------------------------------------------------
# Copyright (c) 2014 Polytechnique Montreal <www.neuro.polymtl.ca>
# Author: Augustin Roux
# modified: 2014/09/28
#
# About the license: see the file LICENSE.TXT
#########################################################################################
# TODO: make it compatible with isct_test_function
# TODO: add log file

import sct_utils as sct
from pandas import DataFrame
import time

def test(path_data='', parameters=''):

    # initializations
    result_rmse = float('NaN')
    result_dist_max = float('NaN')
    time_start = time.time()

    if not parameters:
        parameters = '-i t2/t2_seg.nii.gz'

    # parameters
    folder_data = 't2/'
    file_data = 't2_seg.nii.gz'

    # define command
    cmd = 'sct_process_segmentation -i ' + path_data + folder_data + file_data \
          + ' -p centerline' \
          + ' -v 1'
    output = '\n====================================================================================================\n'+cmd+'\n====================================================================================================\n\n'  # copy command
    status, o = sct.run(cmd, 0)
    output += o

    # define command
    cmd = 'sct_process_segmentation -i ' + path_data + folder_data + file_data \
          + ' -p length'
    output = '\n====================================================================================================\n'+cmd+'\n====================================================================================================\n\n'  # copy command
    status, o = sct.run(cmd, 0)
    output += o

    # define command
    # cmd = 'sct_process_segmentation -i ' + path_data + folder_data + file_data \
    #       + ' -p csa' \
    #       + ' -size 1'\
    #       + ' -r 0'\
    #       + ' -v 1'
    # output = '\n====================================================================================================\n'+cmd+'\n====================================================================================================\n\n'  # copy command
    # status, o = sct.run(cmd, 0)
    # output += o

    # CSA with integrity testing on various angles of segmentation
    import numpy as np
    import nibabel as nib
    import pickle
    size_data = 31
    size_seg = 2  # segmentation size in the X and Y dimension is size_seg*2+1
    csa_truevalue = 25.0
    th_csa_error = 1  # maximum threshold of CSA error allowed
    # Initialise numpy volumes
    data_seg = np.zeros((size_data, size_data, size_data), dtype=np.int16)
    # create labels=1 in a shape of a parallelipiped: 5x5xnz
    data_seg[np.round(size_data / 2) - size_seg:np.round(size_data / 2) + size_seg + 1,
             np.round(size_data / 2) - size_seg:np.round(size_data / 2) + size_seg + 1,
             0: size_data] = 1
    # save as nifti
    img = nib.Nifti1Image(data_seg, np.eye(4))
    nib.save(img, 'data_seg.nii.gz')
    # create nifti with rotated header
    # build rotation matrix
    from numpy import matrix, cos, sin
    alpha = 0.175  # =10deg 0.7854  # corresponds to 45deg angle
    beta = 0.175  # 0.7854  # corresponds to 45deg angle
    gamma = 0.175 # 0.7854  # corresponds to 45deg angle
    rotation_matrix = matrix([[cos(alpha)*cos(beta), cos(alpha)*sin(beta)*sin(gamma)-sin(alpha)*cos(gamma), cos(alpha)*sin(beta)*cos(gamma)+sin(alpha)*sin(gamma)],
                              [sin(alpha)*cos(beta), sin(alpha)*sin(beta)*sin(gamma)+cos(alpha)*cos(gamma), sin(alpha)*sin(beta)*cos(gamma)-cos(alpha)*sin(gamma)],
                              [-sin(beta), cos(beta)*sin(gamma), cos(beta)*cos(gamma)]])
    affine_matrix = np.eye(4)
    affine_matrix[0:3, 0:3] = rotation_matrix
    img_rothd = nib.Nifti1Image(data_seg, affine_matrix)
    nib.save(img_rothd, 'data_seg_rothd.nii.gz')
    # rotate src image
    sct.run('sct_label_utils -i data_seg.nii.gz -create 13,13,0,3:13,13,15,2:13,16,15,4:13,13,30,1 -o data_rot_src.nii.gz', 0)
    sct.run('sct_label_utils -i data_seg.nii.gz -create 4,13,6,3:13,13,15,2:13,16,15,4:22,13,26,1 -o data_rot_dest.nii.gz', 0)
    sct.run('sct_register_multimodal -i data_seg.nii.gz -d data_seg.nii.gz -ilabel data_rot_src.nii.gz -dlabel data_rot_dest.nii.gz -param step=0,type=label,dof=Rx_Ry:step=1,type=im,algo=syn,iter=0 -x linear', 0)
    # compute CSA
    cmd = 'sct_process_segmentation -i data_seg.nii.gz' \
          + ' -p csa' \
          + ' -size 0' \
          + ' -r 0' \
          + ' -v 1' \
          + ' -z 5:23' \
          + ' -ofolder csa'
    output = '\n====================================================================================================\n' + cmd + '\n====================================================================================================\n\n'  # copy command
    status, o = sct.run(cmd, 0)
    output += o
    # check integrity
    csa = pickle.load(open('csa/csa_mean.pickle', 'rb'))
    csa_error = np.abs(csa['MEAN across slices'] - csa_truevalue)
    if csa_error > th_csa_error:
        status = 99
        output += '\nWARNING: CSA_ERROR = ' + str(csa_error) + ' < ' + str(th_csa_error)

    # compute CSA on rotated image
    cmd = 'sct_process_segmentation -i data_seg_reg.nii.gz' \
          + ' -p csa' \
          + ' -size 0' \
          + ' -r 0' \
          + ' -v 1' \
          + ' -z 5:23' \
          + ' -ofolder csa_rot'
    output = '\n====================================================================================================\n' + cmd + '\n====================================================================================================\n\n'  # copy command
    status, o = sct.run(cmd, 0)
    output += o
    # check integrity
    csa = pickle.load(open('csa_rot/csa_mean.pickle', 'rb'))
    csa_error = np.abs(csa['MEAN across slices'] - csa_truevalue)
    if csa_error > th_csa_error:
        status = 99
        output += '\nWARNING: CSA_ERROR = ' + str(csa_error) + ' < ' + str(th_csa_error)

    # compute CSA on rotated header
    cmd = 'sct_process_segmentation -i data_seg_rothd.nii.gz' \
          + ' -p csa' \
          + ' -size 0' \
          + ' -r 0' \
          + ' -v 1' \
          + ' -z 5:23' \
          + ' -ofolder csa_rothd'
    output = '\n====================================================================================================\n' + cmd + '\n====================================================================================================\n\n'  # copy command
    status, o = sct.run(cmd, 0)
    output += o
    # check integrity
    csa = pickle.load(open('csa_rothd/csa_mean.pickle', 'rb'))
    csa_error = np.abs(csa['MEAN across slices'] - csa_truevalue)
    if csa_error > th_csa_error:
        status = 99
        output += '\nWARNING: CSA_ERROR = ' + str(csa_error) + ' < ' + str(th_csa_error)

    # transform results into Pandas structure
    duration = time.time() - time_start
    results = DataFrame(data={'status': int(status), 'output': output, 'csa_error': csa_error, 'duration': duration}, index=[path_data])
    return status, output, results

if __name__ == "__main__":
    # call main function
    test()
<|endoftext|>


<commit_before><commit_msg>ob_estagios pega também o nome do operador<commit_after><|endoftext|>


<commit_before>import os
from OpenGL import GL
from PIL import Image

class Texture:
    """Represents a texture"""
    def __init__(self, name=None, path=None, width=0, height=0, depth=0, lod=0, target=GL.GL_TEXTURE_2D,
                 internal_format=GL.GL_RGBA8, format=GL.GL_RGBA, type=GL.GL_UNSIGNED_BYTE,
                 mipmap=False, anisotropy=0, min_filter=GL.GL_LINEAR, mag_filter=GL.GL_LINEAR,
                 wrap_s=GL.GL_CLAMP_TO_EDGE, wrap_t=GL.GL_CLAMP_TO_EDGE, wrap_r=GL.GL_CLAMP_TO_EDGE,
                 ):
        """Initialize texture without allocating data using default values"""
        self.texture = GL.glGenTextures(1)
        # dimensions
        self.width = width
        self.height = height
        self.depth = depth
        # format / type
        self.target = target
        self.lod = lod
        self.internal_format = internal_format
        self.format = format
        self.type = type
        self.wrap_s = wrap_s
        self.wrap_t = wrap_t
        self.wrap_r = wrap_r
        # filters
        self.min_filter = min_filter
        self.mag_filter = mag_filter
        self.anisotropy = anisotropy
        self.mipmap = mipmap
        # For pre-loading files
        self.name = name
        self.path = path

    @property
    def size(self):
        """
        Get the dimensions of the texture

        :return: (w, h) tuple
        """
        return self.width, self.height

    @classmethod
    def from_image(cls, path, image=None, **kwargs):
        """
        Creates and image from a image file using Pillow/PIL.
        Additional parameters is passed to the texture initializer.

        :param path: The path to the file
        :param image: The PIL/Pillow image object (Can be None)
        :return: Texture object
        """
        t = Texture(path=path, name=os.path.basename(path), **kwargs)
        if image:
            t.set_image(image)
        return t

    @classmethod
    def create_2d(cls, **kwargs):
        """
        Creates a 2d texture.
        All parameters are passed on the texture initializer.

        :return: Texture object
        """
        kwargs['target'] = GL.GL_TEXTURE_2D
        t = Texture(**kwargs)
        t._build()
        return t

    def bind(self):
        """
        Binds the texture to the currently active texture unit
        """
        GL.glBindTexture(self.target, self.texture)

    def _build(self, data=None):
        """Internal method for building the texture"""
        self.bind()
        GL.glTexParameteri(self.target, GL.GL_TEXTURE_MIN_FILTER, self.min_filter)
        GL.glTexParameteri(self.target, GL.GL_TEXTURE_MAG_FILTER, self.mag_filter)

        GL.glTexParameteri(self.target, GL.GL_TEXTURE_WRAP_S, self.wrap_s)
        GL.glTexParameteri(self.target, GL.GL_TEXTURE_WRAP_T, self.wrap_t)
        GL.glTexParameteri(self.target, GL.GL_TEXTURE_WRAP_R, self.wrap_r)

        if self.target == GL.GL_TEXTURE_2D:
            GL.glTexImage2D(self.target, self.lod, self.internal_format,
                            self.width, self.height, 0,
                            self.format, self.type, data)
        elif self.target == GL.GL_TEXTURE_1D:
            if self.width > self.height:
                GL.glTexImage1D(self.target, self.lod, self.internal_format,
                                self.width, 0, self.format, self.type, data)
            else:
                GL.glTexImage1D(self.target, self.lod, self.internal_format,
                                self.height, 0, self.format, self.type, data)

    def set_image(self, image):
        """
        Set pixel data using a image file with PIL/Pillow.

        :param image: The PIL/Pillow image object
        """
        image_flipped = image.transpose(Image.FLIP_TOP_BOTTOM)
        data = image_flipped.convert("RGBA").tobytes()
        self.width, self.height = image.size
        if self.width == 1 or self.height == 1:
            self.target = GL.GL_TEXTURE_1D
        else:
            self.target = GL.GL_TEXTURE_2D
        self._build(data=data)

    def set_texture_repeat(self, wrap_s, wrap_t, wrap_r):
        """
        Sets the texture repeat mode

        :param wrap_s: Repeat mode S (glenum)
        :param wrap_t: Repeat mode T (glenum)
        :param wrap_r: Repeat mode R (glenum)
        """
        self.wrap_s = wrap_s
        self.wrap_t = wrap_t
        self.wrap_r = wrap_r

        self.bind()
        GL.glTexParameteri(self.target, GL.GL_TEXTURE_WRAP_S, self.wrap_s)
        GL.glTexParameteri(self.target, GL.GL_TEXTURE_WRAP_T, self.wrap_t)
        GL.glTexParameteri(self.target, GL.GL_TEXTURE_WRAP_R, self.wrap_r)

    def set_interpolation(self, min_filter, mag_filter):
        """
        Sets the texture interpolation mode

        :param min_filter: Min filter mode (glenum)
        :param mag_filter: Max filter mode (glenum)
        """
        self.min_filter = min_filter
        self.mag_filter = mag_filter
        self.bind()
        GL.glTexParameteri(self.target, GL.GL_TEXTURE_MIN_FILTER, self.min_filter)
        GL.glTexParameteri(self.target, GL.GL_TEXTURE_MAG_FILTER, self.mag_filter)
<commit_msg>Support mipmapping<commit_after>import os
from OpenGL import GL
from PIL import Image

class Texture:
    """Represents a texture"""
    def __init__(self, name=None, path=None, width=0, height=0, depth=0, lod=0, target=GL.GL_TEXTURE_2D,
                 internal_format=GL.GL_RGBA8, format=GL.GL_RGBA, type=GL.GL_UNSIGNED_BYTE,
                 mipmap=False, anisotropy=0, min_filter=GL.GL_LINEAR, mag_filter=GL.GL_LINEAR,
                 wrap_s=GL.GL_CLAMP_TO_EDGE, wrap_t=GL.GL_CLAMP_TO_EDGE, wrap_r=GL.GL_CLAMP_TO_EDGE,
                 ):
        """Initialize texture without allocating data using default values"""
        self.texture = GL.glGenTextures(1)
        # dimensions
        self.width = width
        self.height = height
        self.depth = depth
        # format / type
        self.target = target
        self.lod = lod
        self.internal_format = internal_format
        self.format = format
        self.type = type
        self.wrap_s = wrap_s
        self.wrap_t = wrap_t
        self.wrap_r = wrap_r
        # filters
        self.min_filter = min_filter
        self.mag_filter = mag_filter
        self.anisotropy = anisotropy
        self.mipmap = mipmap
        # Force mipmaps if anisotropy is specified
        if self.anisotropy > 0:
            self.mipmap = True
        # Ensure we are using the right interpolation modes
        if self.mipmap:
            self.min_filter = GL.GL_LINEAR_MIPMAP_LINEAR
            self.mag_filter = GL.GL_LINEAR
        # For pre-loading files
        self.name = name
        self.path = path

    @property
    def size(self):
        """
        Get the dimensions of the texture

        :return: (w, h) tuple
        """
        return self.width, self.height

    @classmethod
    def from_image(cls, path, image=None, **kwargs):
        """
        Creates and image from a image file using Pillow/PIL.
        Additional parameters is passed to the texture initializer.

        :param path: The path to the file
        :param image: The PIL/Pillow image object (Can be None)
        :return: Texture object
        """
        t = Texture(path=path, name=os.path.basename(path), **kwargs)
        if image:
            t.set_image(image)
        return t

    @classmethod
    def create_2d(cls, **kwargs):
        """
        Creates a 2d texture.
        All parameters are passed on the texture initializer.

        :return: Texture object
        """
        kwargs['target'] = GL.GL_TEXTURE_2D
        t = Texture(**kwargs)
        t._build()
        return t

    def bind(self):
        """
        Binds the texture to the currently active texture unit
        """
        GL.glBindTexture(self.target, self.texture)

    def _build(self, data=None):
        """Internal method for building the texture"""
        self.bind()
        GL.glTexParameteri(self.target, GL.GL_TEXTURE_MIN_FILTER, self.min_filter)
        GL.glTexParameteri(self.target, GL.GL_TEXTURE_MAG_FILTER, self.mag_filter)

        GL.glTexParameteri(self.target, GL.GL_TEXTURE_WRAP_S, self.wrap_s)
        GL.glTexParameteri(self.target, GL.GL_TEXTURE_WRAP_T, self.wrap_t)
        GL.glTexParameteri(self.target, GL.GL_TEXTURE_WRAP_R, self.wrap_r)

        if self.target == GL.GL_TEXTURE_2D:
            GL.glTexImage2D(self.target, self.lod, self.internal_format,
                            self.width, self.height, 0,
                            self.format, self.type, data)
        elif self.target == GL.GL_TEXTURE_1D:
            if self.width > self.height:
                GL.glTexImage1D(self.target, self.lod, self.internal_format,
                                self.width, 0, self.format, self.type, data)
            else:
                GL.glTexImage1D(self.target, self.lod, self.internal_format,
                                self.height, 0, self.format, self.type, data)

        if self.mipmap:
            GL.glGenerateMipmap(self.target)

    def set_image(self, image):
        """
        Set pixel data using a image file with PIL/Pillow.

        :param image: The PIL/Pillow image object
        """
        image_flipped = image.transpose(Image.FLIP_TOP_BOTTOM)
        data = image_flipped.convert("RGBA").tobytes()
        self.width, self.height = image.size
        if self.width == 1 or self.height == 1:
            self.target = GL.GL_TEXTURE_1D
        else:
            self.target = GL.GL_TEXTURE_2D
        self._build(data=data)

    def set_texture_repeat(self, wrap_s, wrap_t, wrap_r):
        """
        Sets the texture repeat mode

        :param wrap_s: Repeat mode S (glenum)
        :param wrap_t: Repeat mode T (glenum)
        :param wrap_r: Repeat mode R (glenum)
        """
        self.wrap_s = wrap_s
        self.wrap_t = wrap_t
        self.wrap_r = wrap_r

        self.bind()
        GL.glTexParameteri(self.target, GL.GL_TEXTURE_WRAP_S, self.wrap_s)
        GL.glTexParameteri(self.target, GL.GL_TEXTURE_WRAP_T, self.wrap_t)
        GL.glTexParameteri(self.target, GL.GL_TEXTURE_WRAP_R, self.wrap_r)

    def set_interpolation(self, min_filter, mag_filter):
        """
        Sets the texture interpolation mode

        :param min_filter: Min filter mode (glenum)
        :param mag_filter: Max filter mode (glenum)
        """
        self.min_filter = min_filter
        self.mag_filter = mag_filter
        self.bind()
        GL.glTexParameteri(self.target, GL.GL_TEXTURE_MIN_FILTER, self.min_filter)
        GL.glTexParameteri(self.target, GL.GL_TEXTURE_MAG_FILTER, self.mag_filter)
<|endoftext|>


<commit_before>'''
Manage command to export intervention data for use by others.

Generates a CSV and JSON file with details for all interventions
documented in the database.

Takes an optional argument to specify the output directory. Otherwise,
files are created in the current directory.

'''

import codecs
from collections import OrderedDict
import csv
import json
import os.path
import re

from django.urls import reverse

from derrida.books.management.commands import reference_data
from derrida.interventions.models import Intervention

class Command(reference_data.Command):
    '''Export intervention data from the database as CSV and JSON'''
    help = __doc__

    # NOTE: extending reference_data manage command to inherit
    # flatten_data method; there is more overlap and these scripts
    # could probably be generalized further for re-use

    #: fields for CSV output
    csv_fields = [
        'id', 'book_id', 'book_title', 'book_type', 'page', 'tags', 'text_content',
        'text_language', 'text_language_code', 'text_translation',
        'quote_content', 'quote_language', 'quote_language_code', 'annotator',
        'annotation_region', 'ink', 'page_iiif',
    ]

    #: base filename, for CSV and JSON output
    base_filename = 'annotations'

    def handle(self, *args, **kwargs):
        if kwargs['directory']:
            self.base_filename = os.path.join(kwargs['directory'], self.base_filename)

        # aggregate intervention data to be exported for use in generating
        # CSV and JSON output
        data = [self.annotation_data(intervention)
                   for intervention in Intervention.objects.all()]

        # list of dictionaries can be output as is for JSON export
        with open('{}.json'.format(self.base_filename), 'w') as jsonfile:
            json_data = self.remove_empty_keys(data)
            json.dump(json_data, jsonfile, indent=2)

        # generate CSV export
        with open('{}.csv'.format(self.base_filename), 'w') as csvfile:
            # write utf-8 byte order mark at the beginning of the file
            csvfile.write(codecs.BOM_UTF8.decode())

            csvwriter = csv.DictWriter(csvfile, fieldnames=self.csv_fields)
            csvwriter.writeheader()

            for intervention in data:
                csvwriter.writerow(self.flatten_dict(intervention))

    def annotation_data(self, intervention):
        '''Generate a dictionary of data to export for a single
        :class:`~derrida.books.models.Reference` object'''

        # NOTE: using OrderedDict to ensure JSON output follows logical
        # order in Python < 3.6, where dict order is not guaranteed

        if intervention.canvas:
            canvas_url = reverse('books:canvas-image', kwargs={
                'slug': intervention.work_instance.slug,
                'short_id': intervention.canvas.short_id,
                'mode': 'large',
            })
            iiif_base = ('https://derridas-margins.princeton.edu' + canvas_url).replace('/large/', '/iiif/')
            page_iiif = iiif_base + 'full/500,/0/default.jpg'
        else:
            page_iiif = ''

        # This assumes that any annotation would have a canvas and iiif_base has
        #  already been set.
        if intervention.iiif_image_selection():
            # `canonicalize` makes a request for each item. While iterating,
            #  it may be worth commenting out that method.
            iiif_query = re.sub(
                r'https://iiif-cloud.*intermediate_file/', '',
                str(intervention.iiif_image_selection().canonicalize())
            )
            annotation_region = iiif_base + iiif_query
        else:
            annotation_region = ''

        data = OrderedDict([
            ('id', intervention.get_uri()),
            # every intervention *should* be associated with a book,
            # but possible that some are not
            ('book', OrderedDict([
                ('id', intervention.work_instance.get_uri() if intervention.work_instance else ''),
                ('title', intervention.work_instance.display_title() if intervention.work_instance else ''),
                ('type', intervention.work_instance.item_type if intervention.work_instance else '')
            ])),
            # canvas object *should* have a label, but possible it does not
            ('page', intervention.canvas.label if intervention.canvas else ''),
            ('tags', [tag.name for tag in intervention.tags.all()]),
            ('ink', intervention.ink),
            ('page_iiif', page_iiif),
            ('annotation_region', annotation_region),
        ])

        # only include text and quote information if we have content
        if intervention.text:
            text_info = OrderedDict({
                'content': intervention.text
            })
            if intervention.text_language:
                text_info['language'] = intervention.text_language.name
                text_info['language_code'] = intervention.text_language.code
            if intervention.text_translation:
                text_info['translation'] = intervention.text_translation

            data['text'] = text_info

        if intervention.quote:
            quote_info = OrderedDict({
                'content': intervention.quote
            })
            if intervention.quote_language:
                quote_info['language'] = intervention.quote_language.name
                quote_info['language_code'] = intervention.quote_language.code
            data['quote'] = quote_info

        if intervention.author:
            data['annotator'] = intervention.author.authorized_name

        return data
<commit_msg>Refactor iiif link export<commit_after>'''
Manage command to export intervention data for use by others.

Generates a CSV and JSON file with details for all interventions
documented in the database.

Takes an optional argument to specify the output directory. Otherwise,
files are created in the current directory.

'''

import codecs
from collections import OrderedDict
import csv
import json
import os.path
import re

from django.urls import reverse

from derrida.common.utils import absolutize_url
from derrida.books.management.commands import reference_data
from derrida.interventions.models import Intervention

class Command(reference_data.Command):
    '''Export intervention data from the database as CSV and JSON'''
    help = __doc__

    # NOTE: extending reference_data manage command to inherit
    # flatten_data method; there is more overlap and these scripts
    # could probably be generalized further for re-use

    #: fields for CSV output
    csv_fields = [
        'id', 'book_id', 'book_title', 'book_type', 'page', 'tags', 'text_content',
        'text_language', 'text_language_code', 'text_translation',
        'quote_content', 'quote_language', 'quote_language_code', 'annotator',
        'annotation_region', 'ink', 'page_iiif',
    ]

    #: base filename, for CSV and JSON output
    base_filename = 'annotations'

    def handle(self, *args, **kwargs):
        if kwargs['directory']:
            self.base_filename = os.path.join(kwargs['directory'], self.base_filename)

        # aggregate intervention data to be exported for use in generating
        # CSV and JSON output
        data = [self.annotation_data(intervention)
                   for intervention in Intervention.objects.all()]

        # list of dictionaries can be output as is for JSON export
        with open('{}.json'.format(self.base_filename), 'w') as jsonfile:
            json_data = self.remove_empty_keys(data)
            json.dump(json_data, jsonfile, indent=2)

        # generate CSV export
        with open('{}.csv'.format(self.base_filename), 'w') as csvfile:
            # write utf-8 byte order mark at the beginning of the file
            csvfile.write(codecs.BOM_UTF8.decode())

            csvwriter = csv.DictWriter(csvfile, fieldnames=self.csv_fields)
            csvwriter.writeheader()

            for intervention in data:
                csvwriter.writerow(self.flatten_dict(intervention))

    def annotation_data(self, intervention):
        '''Generate a dictionary of data to export for a single
        :class:`~derrida.books.models.Reference` object'''

        # NOTE: using OrderedDict to ensure JSON output follows logical
        # order in Python < 3.6, where dict order is not guaranteed

        page_iiif = annotation_region_url = None

        if intervention.canvas:
            # get the piffle image api client for the canvas
            img = intervention.canvas.image
            # replace original iiif api endpoint and image id with local proxy iiif url
            img.api_endpoint = absolutize_url(reverse('books:canvas-detail', kwargs={'slug': intervention.work_instance.slug, 'short_id': 'images'})).rstrip('/')
            img.image_id = '%s/iiif' % intervention.canvas.short_id
            # request 500 width image and convert to str for iiif image url
            page_iiif = str(img.size(width=500))

            # get the annotation region
            annotation_region = intervention.iiif_image_selection()
            if annotation_region:
                # `canonicalize` makes a request for each item. While iterating,
                #  it may be worth commenting out that method.
                region = annotation_region.canonicalize()
                region.api_endpoint = img.api_endpoint
                region.image_id = img.image_id
                annotation_region_url = str(region)

        data = OrderedDict([
            ('id', intervention.get_uri()),
            # every intervention *should* be associated with a book,
            # but possible that some are not
            ('book', OrderedDict([
                ('id', intervention.work_instance.get_uri() if intervention.work_instance else ''),
                ('title', intervention.work_instance.display_title() if intervention.work_instance else ''),
                ('type', intervention.work_instance.item_type if intervention.work_instance else '')
            ])),
            # canvas object *should* have a label, but possible it does not
            ('page', intervention.canvas.label if intervention.canvas else ''),
            ('tags', [tag.name for tag in intervention.tags.all()]),
            ('ink', intervention.ink),
            ('page_iiif', page_iiif),
            ('annotation_region', annotation_region_url),
        ])

        # only include text and quote information if we have content
        if intervention.text:
            text_info = OrderedDict({
                'content': intervention.text
            })
            if intervention.text_language:
                text_info['language'] = intervention.text_language.name
                text_info['language_code'] = intervention.text_language.code
            if intervention.text_translation:
                text_info['translation'] = intervention.text_translation

            data['text'] = text_info

        if intervention.quote:
            quote_info = OrderedDict({
                'content': intervention.quote
            })
            if intervention.quote_language:
                quote_info['language'] = intervention.quote_language.name
                quote_info['language_code'] = intervention.quote_language.code
            data['quote'] = quote_info

        if intervention.author:
            data['annotator'] = intervention.author.authorized_name

        return data
<|endoftext|>


<commit_before># Copyright (C) 2019-2020 VyOS maintainers and contributors
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 2 or later as
# published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

import os
import unittest
import json

from netifaces import ifaddresses, AF_INET, AF_INET6

from vyos.configsession import ConfigSession
from vyos.ifconfig import Interface
from vyos.util import read_file
from vyos.util import cmd
from vyos.util import dict_search
from vyos.validate import is_intf_addr_assigned
from vyos.validate import is_ipv6_link_local

class BasicInterfaceTest:
    class BaseTest(unittest.TestCase):
        _test_ip = False
        _test_mtu = False
        _test_vlan = False
        _test_qinq = False
        _test_ipv6 = False
        _test_mirror = False
        _base_path = []

        _options = {}
        _interfaces = []
        _qinq_range = ['10', '20', '30']
        _vlan_range = ['100', '200', '300', '2000']
        # choose IPv6 minimum MTU value for tests - this must always work
        _mtu = '1280'

        def setUp(self):
            self.session = ConfigSession(os.getpid())

            self._test_addr = ['192.0.2.1/26', '192.0.2.255/31', '192.0.2.64/32',
                                '2001:db8:1::ffff/64', '2001:db8:101::1/112']
            self._test_mtu = False
            self._options = {}

        def tearDown(self):
            # we should not remove ethernet from the overall CLI
            if 'ethernet' in self._base_path:
                for interface in self._interfaces:
                    # when using a dedicated interface to test via TEST_ETH environment
                    # variable only this one will be cleared in the end - usable to test
                    # ethernet interfaces via SSH
                    self.session.delete(self._base_path + [interface])
                    self.session.set(self._base_path + [interface, 'duplex', 'auto'])
                    self.session.set(self._base_path + [interface, 'speed', 'auto'])
            else:
                self.session.delete(self._base_path)

            self.session.commit()
            del self.session

        def test_mirror(self):
            Success = 0
            i = 0
            if self._test_mirror:
                for interface in self._interfaces:
                    self.session.set(self._base_path + [interface, 'mirror', 'lo'])
                    i+=1
                self.session.commit()
                # Parse configuration
                for interface in self._interfaces:
                    get_tc_cmd = 'tc -j qdisc'
                    tmp = cmd(get_tc_cmd, shell=True)
                    data = json.loads(tmp)
                    for rule in data:
                        dev = rule['dev']
                        handle = rule['handle']
                        kind = rule['kind']
                        if dev == interface and handle == "ffff:" and kind == "ingress":
                            Success+=1
                if Success == i:
                    self.assertTrue(True)
                else:
                    self.assertTrue(False)
            else:
                return None

        def test_add_description(self):
            """
            Check if description can be added to interface
            """
            for intf in self._interfaces:
                test_string='Description-Test-{}'.format(intf)
                self.session.set(self._base_path + [intf, 'description', test_string])
                for option in self._options.get(intf, []):
                    self.session.set(self._base_path + [intf] + option.split())

            self.session.commit()

            # Validate interface description
            for intf in self._interfaces:
                test_string='Description-Test-{}'.format(intf)
                with open('/sys/class/net/{}/ifalias'.format(intf), 'r') as f:
                    tmp = f.read().rstrip()
                    self.assertTrue(tmp, test_string)

        def test_add_address_single(self):
            """
            Check if a single address can be added to interface.
            """
            addr = '192.0.2.0/31'
            for intf in self._interfaces:
                self.session.set(self._base_path + [intf, 'address', addr])
                for option in self._options.get(intf, []):
                    self.session.set(self._base_path + [intf] + option.split())

            self.session.commit()

            for intf in self._interfaces:
                self.assertTrue(is_intf_addr_assigned(intf, addr))

        def test_add_address_multi(self):
            """
            Check if IPv4/IPv6 addresses can be added to interface.
            """

            # Add address
            for intf in self._interfaces:
                for addr in self._test_addr:
                    self.session.set(self._base_path + [intf, 'address', addr])
                    for option in self._options.get(intf, []):
                        self.session.set(self._base_path + [intf] + option.split())

            self.session.commit()

            # Validate address
            for intf in self._interfaces:
                for af in AF_INET, AF_INET6:
                    for addr in ifaddresses(intf)[af]:
                        # checking link local addresses makes no sense
                        if is_ipv6_link_local(addr['addr']):
                            continue

                        self.assertTrue(is_intf_addr_assigned(intf, addr['addr']))

        def test_ipv6_link_local(self):
            """ Common function for IPv6 link-local address assignemnts """
            if not self._test_ipv6:
                return None

            for interface in self._interfaces:
                base = self._base_path + [interface]
                for option in self._options.get(interface, []):
                    self.session.set(base + option.split())

            # after commit we must have an IPv6 link-local address
            self.session.commit()

            for interface in self._interfaces:
                for addr in ifaddresses(interface)[AF_INET6]:
                    self.assertTrue(is_ipv6_link_local(addr['addr']))

            # disable IPv6 link-local address assignment
            for interface in self._interfaces:
                base = self._base_path + [interface]
                self.session.set(base + ['ipv6', 'address', 'no-default-link-local'])

            # after commit we must have no IPv6 link-local address
            self.session.commit()

            for interface in self._interfaces:
                self.assertTrue(AF_INET6 not in ifaddresses(interface))

        def _mtu_test(self, intf):
            """ helper function to verify MTU size """
            with open('/sys/class/net/{}/mtu'.format(intf), 'r') as f:
                tmp = f.read().rstrip()
                self.assertEqual(tmp, self._mtu)

        def test_change_mtu(self):
            """ Testcase if MTU can be changed on interface """
            if not self._test_mtu:
                return None

            for intf in self._interfaces:
                base = self._base_path + [intf]
                self.session.set(base + ['mtu', self._mtu])
                for option in self._options.get(intf, []):
                    self.session.set(base + option.split())

            # commit interface changes
            self.session.commit()

            # verify changed MTU
            for intf in self._interfaces:
                self._mtu_test(intf)

        def test_change_mtu_1200(self):
            """ Testcase if MTU can be changed to 1200 on non IPv6 enabled interfaces """
            if not self._test_mtu:
                return None

            old_mtu = self._mtu
            self._mtu = '1200'

            for intf in self._interfaces:
                base = self._base_path + [intf]
                self.session.set(base + ['mtu', self._mtu])
                self.session.set(base + ['ipv6', 'address', 'no-default-link-local'])

                for option in self._options.get(intf, []):
                    self.session.set(base + option.split())

            # commit interface changes
            self.session.commit()

            # verify changed MTU
            for intf in self._interfaces:
                self._mtu_test(intf)

            self._mtu = old_mtu

        def test_8021q_vlan(self):
            """ Testcase for 802.1q VLAN interfaces """
            if not self._test_vlan:
                return None

            for interface in self._interfaces:
                base = self._base_path + [interface]
                for option in self._options.get(interface, []):
                    self.session.set(base + option.split())

                for vlan in self._vlan_range:
                    base = self._base_path + [interface, 'vif', vlan]
                    self.session.set(base + ['mtu', self._mtu])
                    for address in self._test_addr:
                        self.session.set(base + ['address', address])

            self.session.commit()
            for intf in self._interfaces:
                for vlan in self._vlan_range:
                    vif = f'{intf}.{vlan}'
                    for address in self._test_addr:
                        self.assertTrue(is_intf_addr_assigned(vif, address))
                    self._mtu_test(vif)

        def test_8021ad_qinq_vlan(self):
            """ Testcase for 802.1ad Q-in-Q VLAN interfaces """
            if not self._test_qinq:
                return None

            for interface in self._interfaces:
                base = self._base_path + [interface]
                for option in self._options.get(interface, []):
                    self.session.set(base + option.split())

                for vif_s in self._qinq_range:
                    for vif_c in self._vlan_range:
                        base = self._base_path + [interface, 'vif-s', vif_s, 'vif-c', vif_c]
                        self.session.set(base + ['mtu', self._mtu])
                        for address in self._test_addr:
                            self.session.set(base + ['address', address])

            self.session.commit()

            for interface in self._interfaces:
                for vif_s in self._qinq_range:
                    tmp = json.loads(cmd(f'ip -d -j link show dev {interface}.{vif_s}'))[0]
                    self.assertEqual(dict_search('linkinfo.info_data.protocol', tmp), '802.1ad')

                    for vif_c in self._vlan_range:
                        vif = f'{interface}.{vif_s}.{vif_c}'
                        for address in self._test_addr:
                            self.assertTrue(is_intf_addr_assigned(vif, address))
                        self._mtu_test(vif)

        def test_ip_options(self):
            """ Test interface base IPv4 options """
            if not self._test_ip:
                return None

            for interface in self._interfaces:
                arp_tmo = '300'
                path = self._base_path + [interface]
                for option in self._options.get(interface, []):
                    self.session.set(path + option.split())

                # Options
                self.session.set(path + ['ip', 'arp-cache-timeout', arp_tmo])
                self.session.set(path + ['ip', 'disable-arp-filter'])
                self.session.set(path + ['ip', 'disable-forwarding'])
                self.session.set(path + ['ip', 'enable-arp-accept'])
                self.session.set(path + ['ip', 'enable-arp-announce'])
                self.session.set(path + ['ip', 'enable-arp-ignore'])
                self.session.set(path + ['ip', 'enable-proxy-arp'])
                self.session.set(path + ['ip', 'proxy-arp-pvlan'])
                self.session.set(path + ['ip', 'source-validation', 'loose'])

            self.session.commit()

            for interface in self._interfaces:
                tmp = read_file(f'/proc/sys/net/ipv4/neigh/{interface}/base_reachable_time_ms')
                self.assertEqual(tmp, str((int(arp_tmo) * 1000))) # tmo value is in milli seconds

                tmp = read_file(f'/proc/sys/net/ipv4/conf/{interface}/arp_filter')
                self.assertEqual('0', tmp)

                tmp = read_file(f'/proc/sys/net/ipv4/conf/{interface}/arp_accept')
                self.assertEqual('1', tmp)

                tmp = read_file(f'/proc/sys/net/ipv4/conf/{interface}/arp_announce')
                self.assertEqual('1', tmp)

                tmp = read_file(f'/proc/sys/net/ipv4/conf/{interface}/arp_ignore')
                self.assertEqual('1', tmp)

                tmp = read_file(f'/proc/sys/net/ipv4/conf/{interface}/forwarding')
                self.assertEqual('0', tmp)

                tmp = read_file(f'/proc/sys/net/ipv4/conf/{interface}/proxy_arp')
                self.assertEqual('1', tmp)

                tmp = read_file(f'/proc/sys/net/ipv4/conf/{interface}/proxy_arp_pvlan')
                self.assertEqual('1', tmp)

                tmp = read_file(f'/proc/sys/net/ipv4/conf/{interface}/rp_filter')
                self.assertEqual('2', tmp)

        def test_ipv6_options(self):
            """ Test interface base IPv6 options """
            if not self._test_ipv6:
                return None

            for interface in self._interfaces:
                dad_transmits = '10'
                path = self._base_path + [interface]
                for option in self._options.get(interface, []):
                    self.session.set(path + option.split())

                # Options
                self.session.set(path + ['ipv6', 'disable-forwarding'])
                self.session.set(path + ['ipv6', 'dup-addr-detect-transmits', dad_transmits])

            self.session.commit()

            for interface in self._interfaces:
                tmp = read_file(f'/proc/sys/net/ipv6/conf/{interface}/forwarding')
                self.assertEqual('0', tmp)

                tmp = read_file(f'/proc/sys/net/ipv6/conf/{interface}/dad_transmits')
                self.assertEqual(dad_transmits, tmp)
<commit_msg>smoketest: interface: move to Python3 'f'ormatted string<commit_after># Copyright (C) 2019-2020 VyOS maintainers and contributors
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 2 or later as
# published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

import os
import unittest
import json

from netifaces import ifaddresses
from netifaces import AF_INET
from netifaces import AF_INET6

from vyos.configsession import ConfigSession
from vyos.ifconfig import Interface
from vyos.util import read_file
from vyos.util import cmd
from vyos.util import dict_search
from vyos.validate import is_intf_addr_assigned
from vyos.validate import is_ipv6_link_local

class BasicInterfaceTest:
    class BaseTest(unittest.TestCase):
        _test_ip = False
        _test_mtu = False
        _test_vlan = False
        _test_qinq = False
        _test_ipv6 = False
        _test_mirror = False
        _base_path = []

        _options = {}
        _interfaces = []
        _qinq_range = ['10', '20', '30']
        _vlan_range = ['100', '200', '300', '2000']
        # choose IPv6 minimum MTU value for tests - this must always work
        _mtu = '1280'

        def setUp(self):
            self.session = ConfigSession(os.getpid())

            self._test_addr = ['192.0.2.1/26', '192.0.2.255/31', '192.0.2.64/32',
                                '2001:db8:1::ffff/64', '2001:db8:101::1/112']
            self._test_mtu = False
            self._options = {}

        def tearDown(self):
            # we should not remove ethernet from the overall CLI
            if 'ethernet' in self._base_path:
                for interface in self._interfaces:
                    # when using a dedicated interface to test via TEST_ETH environment
                    # variable only this one will be cleared in the end - usable to test
                    # ethernet interfaces via SSH
                    self.session.delete(self._base_path + [interface])
                    self.session.set(self._base_path + [interface, 'duplex', 'auto'])
                    self.session.set(self._base_path + [interface, 'speed', 'auto'])
            else:
                self.session.delete(self._base_path)

            self.session.commit()
            del self.session

        def test_mirror(self):
            Success = 0
            i = 0
            if self._test_mirror:
                for interface in self._interfaces:
                    self.session.set(self._base_path + [interface, 'mirror', 'lo'])
                    i+=1
                self.session.commit()
                # Parse configuration
                for interface in self._interfaces:
                    get_tc_cmd = 'tc -j qdisc'
                    tmp = cmd(get_tc_cmd, shell=True)
                    data = json.loads(tmp)
                    for rule in data:
                        dev = rule['dev']
                        handle = rule['handle']
                        kind = rule['kind']
                        if dev == interface and handle == "ffff:" and kind == "ingress":
                            Success+=1
                if Success == i:
                    self.assertTrue(True)
                else:
                    self.assertTrue(False)
            else:
                return None

        def test_add_description(self):
            """
            Check if description can be added to interface
            """
            for intf in self._interfaces:
                test_string=f'Description-Test-{intf}'
                self.session.set(self._base_path + [intf, 'description', test_string])
                for option in self._options.get(intf, []):
                    self.session.set(self._base_path + [intf] + option.split())

            self.session.commit()

            # Validate interface description
            for intf in self._interfaces:
                test_string=f'Description-Test-{intf}'
                with open(f'/sys/class/net/{intf}/ifalias', 'r') as f:
                    tmp = f.read().rstrip()
                    self.assertTrue(tmp, test_string)

        def test_add_address_single(self):
            """
            Check if a single address can be added to interface.
            """
            addr = '192.0.2.0/31'
            for intf in self._interfaces:
                self.session.set(self._base_path + [intf, 'address', addr])
                for option in self._options.get(intf, []):
                    self.session.set(self._base_path + [intf] + option.split())

            self.session.commit()

            for intf in self._interfaces:
                self.assertTrue(is_intf_addr_assigned(intf, addr))

        def test_add_address_multi(self):
            """
            Check if IPv4/IPv6 addresses can be added to interface.
            """

            # Add address
            for intf in self._interfaces:
                for addr in self._test_addr:
                    self.session.set(self._base_path + [intf, 'address', addr])
                    for option in self._options.get(intf, []):
                        self.session.set(self._base_path + [intf] + option.split())

            self.session.commit()

            # Validate address
            for intf in self._interfaces:
                for af in AF_INET, AF_INET6:
                    for addr in ifaddresses(intf)[af]:
                        # checking link local addresses makes no sense
                        if is_ipv6_link_local(addr['addr']):
                            continue

                        self.assertTrue(is_intf_addr_assigned(intf, addr['addr']))

        def test_ipv6_link_local(self):
            """ Common function for IPv6 link-local address assignemnts """
            if not self._test_ipv6:
                return None

            for interface in self._interfaces:
                base = self._base_path + [interface]
                for option in self._options.get(interface, []):
                    self.session.set(base + option.split())

            # after commit we must have an IPv6 link-local address
            self.session.commit()

            for interface in self._interfaces:
                for addr in ifaddresses(interface)[AF_INET6]:
                    self.assertTrue(is_ipv6_link_local(addr['addr']))

            # disable IPv6 link-local address assignment
            for interface in self._interfaces:
                base = self._base_path + [interface]
                self.session.set(base + ['ipv6', 'address', 'no-default-link-local'])

            # after commit we must have no IPv6 link-local address
            self.session.commit()

            for interface in self._interfaces:
                self.assertTrue(AF_INET6 not in ifaddresses(interface))

        def _mtu_test(self, intf):
            """ helper function to verify MTU size """
            with open(f'/sys/class/net/{intf}/mtu', 'r') as f:
                tmp = f.read().rstrip()
                self.assertEqual(tmp, self._mtu)

        def test_change_mtu(self):
            """ Testcase if MTU can be changed on interface """
            if not self._test_mtu:
                return None

            for intf in self._interfaces:
                base = self._base_path + [intf]
                self.session.set(base + ['mtu', self._mtu])
                for option in self._options.get(intf, []):
                    self.session.set(base + option.split())

            # commit interface changes
            self.session.commit()

            # verify changed MTU
            for intf in self._interfaces:
                self._mtu_test(intf)

        def test_change_mtu_1200(self):
            """ Testcase if MTU can be changed to 1200 on non IPv6 enabled interfaces """
            if not self._test_mtu:
                return None

            old_mtu = self._mtu
            self._mtu = '1200'

            for intf in self._interfaces:
                base = self._base_path + [intf]
                self.session.set(base + ['mtu', self._mtu])
                self.session.set(base + ['ipv6', 'address', 'no-default-link-local'])

                for option in self._options.get(intf, []):
                    self.session.set(base + option.split())

            # commit interface changes
            self.session.commit()

            # verify changed MTU
            for intf in self._interfaces:
                self._mtu_test(intf)

            self._mtu = old_mtu

        def test_8021q_vlan(self):
            """ Testcase for 802.1q VLAN interfaces """
            if not self._test_vlan:
                return None

            for interface in self._interfaces:
                base = self._base_path + [interface]
                for option in self._options.get(interface, []):
                    self.session.set(base + option.split())

                for vlan in self._vlan_range:
                    base = self._base_path + [interface, 'vif', vlan]
                    self.session.set(base + ['mtu', self._mtu])
                    for address in self._test_addr:
                        self.session.set(base + ['address', address])

            self.session.commit()
            for intf in self._interfaces:
                for vlan in self._vlan_range:
                    vif = f'{intf}.{vlan}'
                    for address in self._test_addr:
                        self.assertTrue(is_intf_addr_assigned(vif, address))
                    self._mtu_test(vif)

        def test_8021ad_qinq_vlan(self):
            """ Testcase for 802.1ad Q-in-Q VLAN interfaces """
            if not self._test_qinq:
                return None

            for interface in self._interfaces:
                base = self._base_path + [interface]
                for option in self._options.get(interface, []):
                    self.session.set(base + option.split())

                for vif_s in self._qinq_range:
                    for vif_c in self._vlan_range:
                        base = self._base_path + [interface, 'vif-s', vif_s, 'vif-c', vif_c]
                        self.session.set(base + ['mtu', self._mtu])
                        for address in self._test_addr:
                            self.session.set(base + ['address', address])

            self.session.commit()

            for interface in self._interfaces:
                for vif_s in self._qinq_range:
                    tmp = json.loads(cmd(f'ip -d -j link show dev {interface}.{vif_s}'))[0]
                    self.assertEqual(dict_search('linkinfo.info_data.protocol', tmp), '802.1ad')

                    for vif_c in self._vlan_range:
                        vif = f'{interface}.{vif_s}.{vif_c}'
                        for address in self._test_addr:
                            self.assertTrue(is_intf_addr_assigned(vif, address))
                        self._mtu_test(vif)

        def test_ip_options(self):
            """ Test interface base IPv4 options """
            if not self._test_ip:
                return None

            for interface in self._interfaces:
                arp_tmo = '300'
                path = self._base_path + [interface]
                for option in self._options.get(interface, []):
                    self.session.set(path + option.split())

                # Options
                self.session.set(path + ['ip', 'arp-cache-timeout', arp_tmo])
                self.session.set(path + ['ip', 'disable-arp-filter'])
                self.session.set(path + ['ip', 'disable-forwarding'])
                self.session.set(path + ['ip', 'enable-arp-accept'])
                self.session.set(path + ['ip', 'enable-arp-announce'])
                self.session.set(path + ['ip', 'enable-arp-ignore'])
                self.session.set(path + ['ip', 'enable-proxy-arp'])
                self.session.set(path + ['ip', 'proxy-arp-pvlan'])
                self.session.set(path + ['ip', 'source-validation', 'loose'])

            self.session.commit()

            for interface in self._interfaces:
                tmp = read_file(f'/proc/sys/net/ipv4/neigh/{interface}/base_reachable_time_ms')
                self.assertEqual(tmp, str((int(arp_tmo) * 1000))) # tmo value is in milli seconds

                tmp = read_file(f'/proc/sys/net/ipv4/conf/{interface}/arp_filter')
                self.assertEqual('0', tmp)

                tmp = read_file(f'/proc/sys/net/ipv4/conf/{interface}/arp_accept')
                self.assertEqual('1', tmp)

                tmp = read_file(f'/proc/sys/net/ipv4/conf/{interface}/arp_announce')
                self.assertEqual('1', tmp)

                tmp = read_file(f'/proc/sys/net/ipv4/conf/{interface}/arp_ignore')
                self.assertEqual('1', tmp)

                tmp = read_file(f'/proc/sys/net/ipv4/conf/{interface}/forwarding')
                self.assertEqual('0', tmp)

                tmp = read_file(f'/proc/sys/net/ipv4/conf/{interface}/proxy_arp')
                self.assertEqual('1', tmp)

                tmp = read_file(f'/proc/sys/net/ipv4/conf/{interface}/proxy_arp_pvlan')
                self.assertEqual('1', tmp)

                tmp = read_file(f'/proc/sys/net/ipv4/conf/{interface}/rp_filter')
                self.assertEqual('2', tmp)

        def test_ipv6_options(self):
            """ Test interface base IPv6 options """
            if not self._test_ipv6:
                return None

            for interface in self._interfaces:
                dad_transmits = '10'
                path = self._base_path + [interface]
                for option in self._options.get(interface, []):
                    self.session.set(path + option.split())

                # Options
                self.session.set(path + ['ipv6', 'disable-forwarding'])
                self.session.set(path + ['ipv6', 'dup-addr-detect-transmits', dad_transmits])

            self.session.commit()

            for interface in self._interfaces:
                tmp = read_file(f'/proc/sys/net/ipv6/conf/{interface}/forwarding')
                self.assertEqual('0', tmp)

                tmp = read_file(f'/proc/sys/net/ipv6/conf/{interface}/dad_transmits')
                self.assertEqual(dad_transmits, tmp)
<|endoftext|>


<commit_before>
#  IATI Data Quality, tools for Data QA on IATI-formatted  publications
#  by Mark Brough, Martin Keegan, Ben Webb and Jennifer Smith
#
#  Copyright (C) 2013  Publish What You Fund
#
#  This programme is free software; you may redistribute and/or modify
#  it under the terms of the GNU Affero General Public License v3.0

from iatidq import db
import models

class TestNotFound(Exception): pass

def tests(test_id=None):
    if test_id is not None:
        checkTests = models.Test.query.filter_by(id=test_id).first_or_404()
    else:
        checkTests = models.Test.query.order_by(models.Test.id).all()

    if checkTests:
        return checkTests
    else:
        return False

def test_by_test_name(test_name=None):
    checkTest = models.Test.query.filter_by(name=test_name).first()
    if not checkTest:
        raise TestNotFound
    return checkTest

def updateTest(data):
    checkTest = tests(id=data['id'])
    if checkTest:
        for k, v in data.items():
            setattr(checkTest, k, v)
        db.session.add(checkTest)
        db.session.commit()
        return checkTest
    else:
        return False

def deleteTest(test_id):
    checkTest = tests(test_id)
    db.session.delete(checkTest)
    db.session.commit()

def addTest(data):
    try:
        checkTest = test_by_test_name(data["name"])
    except TestNotFound:
        return False

    if not checkTest:
        test = models.Test()
        test.setup(
            name = data['name'],
            description = data['description'],
            test_group = "",
            test_level = data['test_level'],
            active = data['active']
            )
        db.session.add(test)
        db.session.commit()
        return test
    else:
        return False
<commit_msg>remove uncalled branch<commit_after>
#  IATI Data Quality, tools for Data QA on IATI-formatted  publications
#  by Mark Brough, Martin Keegan, Ben Webb and Jennifer Smith
#
#  Copyright (C) 2013  Publish What You Fund
#
#  This programme is free software; you may redistribute and/or modify
#  it under the terms of the GNU Affero General Public License v3.0

from iatidq import db
import models

class TestNotFound(Exception): pass

def tests(test_id=None):
    if test_id is not None:
        checkTests = models.Test.query.filter_by(id=test_id).first_or_404()
    else:
        checkTests = models.Test.query.order_by(models.Test.id).all()

    if checkTests:
        return checkTests
    else:
        return False

def test_by_test_name(test_name=None):
    checkTest = models.Test.query.filter_by(name=test_name).first()
    if not checkTest:
        raise TestNotFound
    return checkTest

def updateTest(data):
    checkTest = tests(id=data['id'])
    if checkTest:
        for k, v in data.items():
            setattr(checkTest, k, v)
        db.session.add(checkTest)
        db.session.commit()
        return checkTest
    else:
        return False

def deleteTest(test_id):
    checkTest = tests(test_id)
    db.session.delete(checkTest)
    db.session.commit()

def addTest(data):
    try:
        checkTest = test_by_test_name(data["name"])
    except TestNotFound:
        return False

    test = models.Test()
    test.setup(
        name = data['name'],
        description = data['description'],
        test_group = "",
        test_level = data['test_level'],
        active = data['active']
        )
    db.session.add(test)
    db.session.commit()
    return test
<|endoftext|>


<commit_before># ######################################################################
# Copyright (c) 2014, Brookhaven Science Associates, Brookhaven        #
# National Laboratory. All rights reserved.                            #
#                                                                      #
# Redistribution and use in source and binary forms, with or without   #
# modification, are permitted provided that the following conditions   #
# are met:                                                             #
#                                                                      #
# * Redistributions of source code must retain the above copyright     #
#   notice, this list of conditions and the following disclaimer.      #
#                                                                      #
# * Redistributions in binary form must reproduce the above copyright  #
#   notice this list of conditions and the following disclaimer in     #
#   the documentation and/or other materials provided with the         #
#   distribution.                                                      #
#                                                                      #
# * Neither the name of the Brookhaven Science Associates, Brookhaven  #
#   National Laboratory nor the names of its contributors may be used  #
#   to endorse or promote products derived from this software without  #
#   specific prior written permission.                                 #
#                                                                      #
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS  #
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT    #
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS    #
# FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE       #
# COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT,           #
# INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES   #
# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR   #
# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)   #
# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,  #
# STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OTHERWISE) ARISING   #
# IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE   #
# POSSIBILITY OF SUCH DAMAGE.                                          #
########################################################################
"""
This module contains code for extracting features from data
"""
from __future__ import (absolute_import, division, print_function,
                        unicode_literals)
import logging
logger = logging.getLogger(__name__)

import six
from six.moves import zip
import numpy as np

from collections import deque

from .fitting import fit_quad_to_peak

class PeakRejection(Exception):
    """Custom exception class to indicate that the refine function rejected
    the candidate peak.

    This uses the exception handling framework in a method akin to
    `StopIteration` to indicate that there will be no return value.
    """
    pass

def peak_refinement(x, y, cands, window, refine_function, refine_args=None):
    """Refine candidate locations

    Parameters
    ----------
    x : array
        The independent variable, does not need to be evenly spaced.

    y : array
        The dependent variable.  Must correspond 1:1 with the values in `x`

    cands : array
        Array of the indices in `x` (and `y`) for the candidate peaks.

    refine_function : function
        A function which takes a section of data with a peak in it and returns
        the location and height of the peak to sub-sample accuracy.  Additional
        parameters can be passed through via the refine_args kwarg.
        The function signature must be::

            center, height = refine_func(x, y, **kwargs)

        This function may raise `PeakRejection` to indicate no suitable
        peak was found

    window : int
        How many samples to extract on either side of the
        candidate locations are passed to the refine function.  The
        window will be truncated near the boundaries.  The length of the
        data passed to the refine function will be (2 * window + 1).

    refine_args : dict, optional
        The passed to the refine_function

    Returns
    -------
    peak_locations : array
        The locations of the peaks

    peak_heights : array
        The heights of the peaks

    Examples
    --------
    >>> x = np.arange(512)
    >>> tt = np.zeros(512)
    >>> tt += np.exp(-((x - 150.55)/10)**2)
    >>> tt += np.exp(-((x - 450.75)/10)**2)
    >>> cands = scipy.signal.argrelmax(tt)[0]

    >>> print(peak_refinement(x, tt, cands, 10, refine_quadratic))
    (array([ 150.62286432,  450.7909412 ]), array([ 0.96435832,  0.96491501]))
    >>> print(peak_refinement(x, tt, cands, 10, refine_log_quadratic))
    (array([ 150.55,  450.75]), array([ 1.,  1.]))
    """
    # clean up input
    x = np.asarray(x)
    y = np.asarray(y)
    cands = np.asarray(cands, dtype=int)
    window = int(window)
    if refine_args is None:
        refine_args = dict()
    # local working variables
    out_tmp = deque()
    max_ind = len(x)

    for ind in cands:
        slc = slice(np.max([0, ind-window]),
                    np.min([max_ind, ind + window + 1]))
        try:
            ret = refine_function(x[slc], y[slc], **refine_args)
        except PeakRejection:
            # We are catching the PeakRejections raised here as
            # an indication that no suitable peak was found
            continue
        else:
            out_tmp.append(ret)

    return tuple([np.array(_) for _ in zip(*out_tmp)])

def refine_quadratic(x, y, Rval_thresh=None):
    """
    Attempts to refine the peaks by fitting to
    a quadratic.

    Parameters
    ----------
    x : array
        Independent variable

    y : array
        Dependent variable

    Rval_thresh : float, optional
        Threshold for R^2 value of fit,  If the computed R^2 is worse than
        this threshold PeakRejection will be raised

    Returns
    -------
    center : float
        Refined estimate for center

    height : float
        Refined estimate for height

    Raises
    ------
    PeakRejection
       Raised to indicate that no suitable peak was found in the
       interval

    """
    beta, R2 = fit_quad_to_peak(x, y)
    if Rval_thresh is not None:
        if R2 < Rval_thresh:
            raise PeakRejection()
    return beta[1], beta[2]

def refine_log_quadratic(x, y, Rval_thresh=None):
    """
    Attempts to refine the peaks by fitting a quadratic to the log of
    the y-data.  This is a linear approximation of fitting a Gaussian.

    Parameters
    ----------
    x : array
        Independent variable

    y : array
        Dependent variable

    Rval_thresh : float, optional
        Threshold for R^2 value of fit,  If the computed R^2 is worse than
        this threshold PeakRejection will be raised

    Returns
    -------
    center : float
        Refined estimate for center

    height : float
        Refined estimate for height

    Raises
    ------
    PeakRejection
       Raised to indicate that no suitable peak was found in the
       interval

    """
    beta, R2 = fit_quad_to_peak(x, np.log(y))
    if Rval_thresh is not None:
        if R2 < Rval_thresh:
            raise PeakRejection()
    return beta[1], np.exp(beta[2])

def filter_n_largest(y, cands, N):
    """Filters the N largest candidate peaks

    Return a maximum of N largest candidates.  If N > len(cands) then
    all of the cands will be returned sorted, else the indices
    of the N largest peaks will be returned in descending order.

    Parameters
    ----------
    y : array
        Independent variable

    cands : array
        An array containing the indices of candidate peaks

    N : int
        The maximum number of peaks to return, sorted by size

    Returns
    -------
    cands : array
        An array of the indices of up to the N largest candidates
    """
    cands = np.asarray(cands)
    N = int(N)
    if N < 0:
        raise ValueError("The maximum number of peaks to return must "
                         "be positive not {}".format(N))

    sorted_args = np.argsort(y[cands])
    # cut out if asking for more peaks than exist
    if len(cands) < N:
        return cands[sorted_args][::-1]

    return cands[sorted_args[-N:]][::-1]

def filter_peak_height(y, cands, thresh, window=5):
    """
    Filter to remove candidate that are too small.  This
    is implemented by looking at the relative height (max - min)
    of the peak in a window around the candidate peak.

    Parameters
    ----------
    y : array
        Independent variable

    cands : array
        An array containing the indices of candidate peaks

    thresh : int
        The minimum peak-to-peak size of the candidate peak to be accepted

    window : int, optional
        The size of the window around the peak to consider

    Returns
    -------
    cands : array
        An array of the indices which pass the filter

    """
    y = np.asarray(y)
    out_tmp = deque()
    max_ind = len(y)
    for ind in cands:
        slc = slice(np.max([0, ind-window]),
                    np.min([max_ind, ind + window + 1]))
        pk_hght = np.ptp(y[slc])
        if pk_hght > thresh:
            out_tmp.append(ind)

    return np.array(out_tmp)
<commit_msg>MNT : simplified if staments<commit_after># ######################################################################
# Copyright (c) 2014, Brookhaven Science Associates, Brookhaven        #
# National Laboratory. All rights reserved.                            #
#                                                                      #
# Redistribution and use in source and binary forms, with or without   #
# modification, are permitted provided that the following conditions   #
# are met:                                                             #
#                                                                      #
# * Redistributions of source code must retain the above copyright     #
#   notice, this list of conditions and the following disclaimer.      #
#                                                                      #
# * Redistributions in binary form must reproduce the above copyright  #
#   notice this list of conditions and the following disclaimer in     #
#   the documentation and/or other materials provided with the         #
#   distribution.                                                      #
#                                                                      #
# * Neither the name of the Brookhaven Science Associates, Brookhaven  #
#   National Laboratory nor the names of its contributors may be used  #
#   to endorse or promote products derived from this software without  #
#   specific prior written permission.                                 #
#                                                                      #
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS  #
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT    #
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS    #
# FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE       #
# COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT,           #
# INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES   #
# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR   #
# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)   #
# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,  #
# STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OTHERWISE) ARISING   #
# IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE   #
# POSSIBILITY OF SUCH DAMAGE.                                          #
########################################################################
"""
This module contains code for extracting features from data
"""
from __future__ import (absolute_import, division, print_function,
                        unicode_literals)
import logging
logger = logging.getLogger(__name__)

import six
from six.moves import zip
import numpy as np

from collections import deque

from .fitting import fit_quad_to_peak

class PeakRejection(Exception):
    """Custom exception class to indicate that the refine function rejected
    the candidate peak.

    This uses the exception handling framework in a method akin to
    `StopIteration` to indicate that there will be no return value.
    """
    pass

def peak_refinement(x, y, cands, window, refine_function, refine_args=None):
    """Refine candidate locations

    Parameters
    ----------
    x : array
        The independent variable, does not need to be evenly spaced.

    y : array
        The dependent variable.  Must correspond 1:1 with the values in `x`

    cands : array
        Array of the indices in `x` (and `y`) for the candidate peaks.

    refine_function : function
        A function which takes a section of data with a peak in it and returns
        the location and height of the peak to sub-sample accuracy.  Additional
        parameters can be passed through via the refine_args kwarg.
        The function signature must be::

            center, height = refine_func(x, y, **kwargs)

        This function may raise `PeakRejection` to indicate no suitable
        peak was found

    window : int
        How many samples to extract on either side of the
        candidate locations are passed to the refine function.  The
        window will be truncated near the boundaries.  The length of the
        data passed to the refine function will be (2 * window + 1).

    refine_args : dict, optional
        The passed to the refine_function

    Returns
    -------
    peak_locations : array
        The locations of the peaks

    peak_heights : array
        The heights of the peaks

    Examples
    --------
    >>> x = np.arange(512)
    >>> tt = np.zeros(512)
    >>> tt += np.exp(-((x - 150.55)/10)**2)
    >>> tt += np.exp(-((x - 450.75)/10)**2)
    >>> cands = scipy.signal.argrelmax(tt)[0]

    >>> print(peak_refinement(x, tt, cands, 10, refine_quadratic))
    (array([ 150.62286432,  450.7909412 ]), array([ 0.96435832,  0.96491501]))
    >>> print(peak_refinement(x, tt, cands, 10, refine_log_quadratic))
    (array([ 150.55,  450.75]), array([ 1.,  1.]))
    """
    # clean up input
    x = np.asarray(x)
    y = np.asarray(y)
    cands = np.asarray(cands, dtype=int)
    window = int(window)
    if refine_args is None:
        refine_args = dict()
    # local working variables
    out_tmp = deque()
    max_ind = len(x)

    for ind in cands:
        slc = slice(np.max([0, ind-window]),
                    np.min([max_ind, ind + window + 1]))
        try:
            ret = refine_function(x[slc], y[slc], **refine_args)
        except PeakRejection:
            # We are catching the PeakRejections raised here as
            # an indication that no suitable peak was found
            continue
        else:
            out_tmp.append(ret)

    return tuple([np.array(_) for _ in zip(*out_tmp)])

def refine_quadratic(x, y, Rval_thresh=None):
    """
    Attempts to refine the peaks by fitting to
    a quadratic.

    Parameters
    ----------
    x : array
        Independent variable

    y : array
        Dependent variable

    Rval_thresh : float, optional
        Threshold for R^2 value of fit,  If the computed R^2 is worse than
        this threshold PeakRejection will be raised

    Returns
    -------
    center : float
        Refined estimate for center

    height : float
        Refined estimate for height

    Raises
    ------
    PeakRejection
       Raised to indicate that no suitable peak was found in the
       interval

    """
    beta, R2 = fit_quad_to_peak(x, y)
    if Rval_thresh is not None and R2 < Rval_thresh:
        raise PeakRejection()

    return beta[1], beta[2]

def refine_log_quadratic(x, y, Rval_thresh=None):
    """
    Attempts to refine the peaks by fitting a quadratic to the log of
    the y-data.  This is a linear approximation of fitting a Gaussian.

    Parameters
    ----------
    x : array
        Independent variable

    y : array
        Dependent variable

    Rval_thresh : float, optional
        Threshold for R^2 value of fit,  If the computed R^2 is worse than
        this threshold PeakRejection will be raised

    Returns
    -------
    center : float
        Refined estimate for center

    height : float
        Refined estimate for height

    Raises
    ------
    PeakRejection
       Raised to indicate that no suitable peak was found in the
       interval

    """
    beta, R2 = fit_quad_to_peak(x, np.log(y))
    if Rval_thresh is not None and R2 < Rval_thresh:
        raise PeakRejection()

    return beta[1], np.exp(beta[2])

def filter_n_largest(y, cands, N):
    """Filters the N largest candidate peaks

    Return a maximum of N largest candidates.  If N > len(cands) then
    all of the cands will be returned sorted, else the indices
    of the N largest peaks will be returned in descending order.

    Parameters
    ----------
    y : array
        Independent variable

    cands : array
        An array containing the indices of candidate peaks

    N : int
        The maximum number of peaks to return, sorted by size

    Returns
    -------
    cands : array
        An array of the indices of up to the N largest candidates
    """
    cands = np.asarray(cands)
    N = int(N)
    if N < 0:
        raise ValueError("The maximum number of peaks to return must "
                         "be positive not {}".format(N))

    sorted_args = np.argsort(y[cands])
    # cut out if asking for more peaks than exist
    if len(cands) < N:
        return cands[sorted_args][::-1]

    return cands[sorted_args[-N:]][::-1]

def filter_peak_height(y, cands, thresh, window=5):
    """
    Filter to remove candidate that are too small.  This
    is implemented by looking at the relative height (max - min)
    of the peak in a window around the candidate peak.

    Parameters
    ----------
    y : array
        Independent variable

    cands : array
        An array containing the indices of candidate peaks

    thresh : int
        The minimum peak-to-peak size of the candidate peak to be accepted

    window : int, optional
        The size of the window around the peak to consider

    Returns
    -------
    cands : array
        An array of the indices which pass the filter

    """
    y = np.asarray(y)
    out_tmp = deque()
    max_ind = len(y)
    for ind in cands:
        slc = slice(np.max([0, ind-window]),
                    np.min([max_ind, ind + window + 1]))
        pk_hght = np.ptp(y[slc])
        if pk_hght > thresh:
            out_tmp.append(ind)

    return np.array(out_tmp)
<|endoftext|>