<commit_before>import numpy as np
import tensorflow as tf
from keras_contrib.layers import InstanceNormalization

PRETRAINED_WEIGHT_DIR = "pretrained_weights"

def conv_layer(style, name, filters, kernel_size, strides=(1, 1), bias=True):
    init_weight = np.load(f"{PRETRAINED_WEIGHT_DIR}/{style}/{name}.weight.npy")
    init_weight = np.transpose(init_weight, [2, 3, 1, 0])
    init_bias = np.load(f"{PRETRAINED_WEIGHT_DIR}/{style}/{name}.bias.npy")

    if bias:
        bias_initializer = tf.keras.initializers.constant(init_bias)
    else:
        bias_initializer = "zeros"

    layer = tf.keras.layers.Conv2D(
        filters=filters,
        kernel_size=kernel_size,
        strides=strides,
        kernel_initializer=tf.keras.initializers.constant(init_weight),
        bias_initializer=bias_initializer
    )
    return layer

def instance_norm_layer(style, name, epsilon=1e-9):
    init_beta = np.load(f"{PRETRAINED_WEIGHT_DIR}/{style}/{name}.shift.npy")
    init_gamma = np.load(f"{PRETRAINED_WEIGHT_DIR}/{style}/{name}.scale.npy")

    layer = InstanceNormalization(
        axis=-1,
        epsilon=epsilon,
        beta_initializer=tf.keras.initializers.constant(init_beta),
        gamma_initializer=tf.keras.initializers.constant(init_gamma)
    )
    return layer

def deconv_layers(style, name, filters, kernel_size, strides=(1, 1)):
    init_weight = np.load(f"{PRETRAINED_WEIGHT_DIR}/{style}/{name}.weight.npy")
    init_weight = np.transpose(init_weight, [2, 3, 1, 0])
    init_bias = np.load(f"{PRETRAINED_WEIGHT_DIR}/{style}/{name}.bias.npy")

    layers = list()
    layers.append(tf.keras.layers.Conv2DTranspose(
        filters=filters,
        kernel_size=kernel_size,
        strides=strides,
        kernel_initializer=tf.keras.initializers.constant(init_weight),
        bias_initializer=tf.keras.initializers.constant(init_bias)
    ))

    layers.append(tf.keras.layers.Cropping2D(cropping=((1, 0), (1, 0))))
    return layers

def build_generator(style):
    inputs = tf.keras.Input(shape=(None, None, 3))

    y = tf.keras.layers.ZeroPadding2D(padding=(3, 3))(inputs)
    y = conv_layer(style, "conv01_1", filters=64, kernel_size=7)(y)
    y = instance_norm_layer(style, "in01_1")(y)
    y = tf.keras.layers.Activation("relu")(y)

    y = tf.keras.layers.ZeroPadding2D(padding=(1, 1))(y)
    y = conv_layer(style, "conv02_1", filters=128, kernel_size=3, strides=(2, 2))(y)
    y = tf.keras.layers.ZeroPadding2D(padding=(1, 1))(y)
    y = conv_layer(style, "conv02_2", filters=128, kernel_size=3, strides=(1, 1))(y)
    y = instance_norm_layer(style, "in02_1")(y)
    y = tf.keras.layers.Activation("relu")(y)

    y = tf.keras.layers.ZeroPadding2D(padding=(1, 1))(y)
    y = conv_layer(style, "conv03_1", filters=256, kernel_size=3, strides=(2, 2))(y)
    y = tf.keras.layers.ZeroPadding2D(padding=(1, 1))(y)
    y = conv_layer(style, "conv03_2", filters=256, kernel_size=3, strides=(1, 1))(y)
    y = instance_norm_layer(style, "in03_1")(y)

    t_prev = tf.keras.layers.Activation("relu")(y)

    for i in range(4, 12):
        y = tf.keras.layers.ZeroPadding2D(padding=(1, 1))(t_prev)
        y = conv_layer(style, "conv%02d_1" % i, filters=256, kernel_size=3)(y)
        y = instance_norm_layer(style, "in%02d_1" % i)(y)
        y = tf.keras.layers.Activation("relu")(y)

        t = tf.keras.layers.ZeroPadding2D(padding=(1, 1))(y)
        t = conv_layer(style, "conv%02d_2" % i, filters=256, kernel_size=3)(t)
        t = instance_norm_layer(style, "in%02d_2" % i)(t)

        t_prev = tf.keras.layers.Add()([t, t_prev])

        if i == 11:
            y = t_prev

    layers = deconv_layers(style, "deconv01_1", filters=128, kernel_size=3, strides=(2, 2))
    for layer in layers:
        y = layer(y)
    y = tf.keras.layers.ZeroPadding2D(padding=(1, 1))(y)
    y = conv_layer(style, "deconv01_2", filters=128, kernel_size=3)(y)
    y = instance_norm_layer(style, "in12_1")(y)
    y = tf.keras.layers.Activation("relu")(y)

    layers = deconv_layers(style, "deconv02_1", filters=64, kernel_size=3, strides=(2, 2))
    for layer in layers:
        y = layer(y)
    y = tf.keras.layers.ZeroPadding2D(padding=(1, 1))(y)
    y = conv_layer(style, "deconv02_2", filters=64, kernel_size=3)(y)
    y = instance_norm_layer(style, "in13_1")(y)
    y = tf.keras.layers.Activation("relu")(y)

    y = tf.keras.layers.ZeroPadding2D(padding=(3, 3))(y)
    y = conv_layer(style, "deconv03_1", filters=3, kernel_size=7)(y)
    y = tf.keras.layers.Activation("tanh")(y)

    model = tf.keras.Model(inputs=inputs, outputs=y)

    return model

if __name__ == '__main__':
    g = build_generator(style="shinkai")
    np.random.seed(9527)
    nx = np.random.rand(1, 225, 225, 3).astype(np.float32)
    out = g(nx)
    tf_out = np.load("tf_out.npy")

    diff = np.sqrt(np.mean((out - tf_out) ** 2))
    assert diff < 1e-6

<commit_msg>feat: add and apply reflection pad layer<commit_after>import numpy as np
import tensorflow as tf
from keras_contrib.layers import InstanceNormalization
from tensorflow.keras.layers import Layer, InputSpec

PRETRAINED_WEIGHT_DIR = "pretrained_weights"

# ref: https://stackoverflow.com/a/53349976/2447655
class ReflectionPadding2D(Layer):
    def __init__(self, padding=(1, 1), **kwargs):
        self.padding = tuple(padding)
        self.input_spec = [InputSpec(ndim=4)]
        super(ReflectionPadding2D, self).__init__(**kwargs)

    def compute_output_shape(self, s):
        """ If you are using "channels_last" configuration"""
        return s[0], s[1] + 2 * self.padding[0], s[2] + 2 * self.padding[1], s[3]

    def call(self, x):
        w_pad, h_pad = self.padding
        return tf.pad(x, [[0, 0], [h_pad, h_pad], [w_pad, w_pad], [0, 0]], 'REFLECT')

def conv_layer(style, name, filters, kernel_size, strides=(1, 1), bias=True):
    init_weight = np.load(f"{PRETRAINED_WEIGHT_DIR}/{style}/{name}.weight.npy")
    init_weight = np.transpose(init_weight, [2, 3, 1, 0])
    init_bias = np.load(f"{PRETRAINED_WEIGHT_DIR}/{style}/{name}.bias.npy")

    if bias:
        bias_initializer = tf.keras.initializers.constant(init_bias)
    else:
        bias_initializer = "zeros"

    layer = tf.keras.layers.Conv2D(
        filters=filters,
        kernel_size=kernel_size,
        strides=strides,
        kernel_initializer=tf.keras.initializers.constant(init_weight),
        bias_initializer=bias_initializer
    )
    return layer

def instance_norm_layer(style, name, epsilon=1e-9):
    init_beta = np.load(f"{PRETRAINED_WEIGHT_DIR}/{style}/{name}.shift.npy")
    init_gamma = np.load(f"{PRETRAINED_WEIGHT_DIR}/{style}/{name}.scale.npy")

    layer = InstanceNormalization(
        axis=-1,
        epsilon=epsilon,
        beta_initializer=tf.keras.initializers.constant(init_beta),
        gamma_initializer=tf.keras.initializers.constant(init_gamma)
    )
    return layer

def deconv_layers(style, name, filters, kernel_size, strides=(1, 1)):
    init_weight = np.load(f"{PRETRAINED_WEIGHT_DIR}/{style}/{name}.weight.npy")
    init_weight = np.transpose(init_weight, [2, 3, 1, 0])
    init_bias = np.load(f"{PRETRAINED_WEIGHT_DIR}/{style}/{name}.bias.npy")

    layers = list()
    layers.append(tf.keras.layers.Conv2DTranspose(
        filters=filters,
        kernel_size=kernel_size,
        strides=strides,
        kernel_initializer=tf.keras.initializers.constant(init_weight),
        bias_initializer=tf.keras.initializers.constant(init_bias)
    ))

    layers.append(tf.keras.layers.Cropping2D(cropping=((1, 0), (1, 0))))
    return layers

def build_model(style):
    inputs = tf.keras.Input(shape=(None, None, 3))

    # y = tf.keras.layers.ZeroPadding2D(padding=(3, 3))(inputs)
    y = ReflectionPadding2D(padding=(3, 3))(inputs)
    y = conv_layer(style, "conv01_1", filters=64, kernel_size=7)(y)
    y = instance_norm_layer(style, "in01_1")(y)
    y = tf.keras.layers.Activation("relu")(y)

    # y = tf.keras.layers.ZeroPadding2D(padding=(1, 1))(y)
    y = ReflectionPadding2D(padding=(1, 1))(y)
    y = conv_layer(style, "conv02_1", filters=128, kernel_size=3, strides=(2, 2))(y)
    # y = tf.keras.layers.ZeroPadding2D(padding=(1, 1))(y)
    y = ReflectionPadding2D(padding=(1, 1))(y)
    y = conv_layer(style, "conv02_2", filters=128, kernel_size=3, strides=(1, 1))(y)
    y = instance_norm_layer(style, "in02_1")(y)
    y = tf.keras.layers.Activation("relu")(y)

    # y = tf.keras.layers.ZeroPadding2D(padding=(1, 1))(y)
    y = ReflectionPadding2D(padding=(1, 1))(y)
    y = conv_layer(style, "conv03_1", filters=256, kernel_size=3, strides=(2, 2))(y)
    # y = tf.keras.layers.ZeroPadding2D(padding=(1, 1))(y)
    y = ReflectionPadding2D(padding=(1, 1))(y)
    y = conv_layer(style, "conv03_2", filters=256, kernel_size=3, strides=(1, 1))(y)
    y = instance_norm_layer(style, "in03_1")(y)

    t_prev = tf.keras.layers.Activation("relu")(y)

    for i in range(4, 12):
        # y = tf.keras.layers.ZeroPadding2D(padding=(1, 1))(t_prev)
        y = ReflectionPadding2D(padding=(1, 1))(t_prev)
        y = conv_layer(style, "conv%02d_1" % i, filters=256, kernel_size=3)(y)
        y = instance_norm_layer(style, "in%02d_1" % i)(y)
        y = tf.keras.layers.Activation("relu")(y)

        t = ReflectionPadding2D(padding=(1, 1))(y)
        t = conv_layer(style, "conv%02d_2" % i, filters=256, kernel_size=3)(t)
        t = instance_norm_layer(style, "in%02d_2" % i)(t)

        t_prev = tf.keras.layers.Add()([t, t_prev])

        if i == 11:
            y = t_prev

    layers = deconv_layers(style, "deconv01_1", filters=128, kernel_size=3, strides=(2, 2))
    for layer in layers:
        y = layer(y)
    # y = tf.keras.layers.ZeroPadding2D(padding=(1, 1))(y)
    y = ReflectionPadding2D(padding=(1, 1))(y)
    y = conv_layer(style, "deconv01_2", filters=128, kernel_size=3)(y)
    y = instance_norm_layer(style, "in12_1")(y)
    y = tf.keras.layers.Activation("relu")(y)

    layers = deconv_layers(style, "deconv02_1", filters=64, kernel_size=3, strides=(2, 2))
    for layer in layers:
        y = layer(y)
    # y = tf.keras.layers.ZeroPadding2D(padding=(1, 1))(y)
    y = ReflectionPadding2D(padding=(1, 1))(y)
    y = conv_layer(style, "deconv02_2", filters=64, kernel_size=3)(y)
    y = instance_norm_layer(style, "in13_1")(y)
    y = tf.keras.layers.Activation("relu")(y)

    # y = tf.keras.layers.ZeroPadding2D(padding=(3, 3))(y)
    y = ReflectionPadding2D(padding=(3, 3))(y)
    y = conv_layer(style, "deconv03_1", filters=3, kernel_size=7)(y)
    y = tf.keras.layers.Activation("tanh")(y)

    model = tf.keras.Model(inputs=inputs, outputs=y)

    return model

if __name__ == '__main__':
    g = build_model(style="shinkai")
    np.random.seed(9527)
    nx = np.random.rand(1, 225, 225, 3).astype(np.float32)
    out = g(nx)
    tf_out = np.load("tf_out.npy")

    diff = np.sqrt(np.mean((out - tf_out) ** 2))
    assert diff < 1e-6

<|endoftext|>


<commit_before>import datetime
from operator import attrgetter, itemgetter

import unittest2 as unittest

from tkp.testutil.decorators import requires_database
from tkp.db.orm import DataSet
from tkp.db.orm import Image
import tkp.db
import tkp.db.transients as dbtransients

class TestLightCurve(unittest.TestCase):
    def setUp(self):
        self.database = tkp.db.Database()
        self.dataset = DataSet(data={'description': 'dataset with images'}, database=self.database)

    def tearDown(self):
        tkp.db.rollback()

    @requires_database()
    def test_lightcurve(self):
        # make 4 * 5 images with different date
        images = []
        for day in [3, 4, 5, 6]:
            data = {'taustart_ts': datetime.datetime(2010, 3, day),
                    'tau_time': 3600,
                    'url': '/',
                    'freq_eff': 80e6,
                    'freq_bw': 1e6,
                    'beam_smaj_pix': float(2.7),
                    'beam_smin_pix': float(2.3),
                    'beam_pa_rad': float(1.7),
                    'deltax': float(-0.01111),
                    'deltay': float(0.01111),
                    'centre_ra': 111,
                    'centre_decl': 11,
                    'xtr_radius' : 3
            }
            image = Image(dataset=self.dataset, data=data)
            images.append(image)

        # 3 sources per image, with different coordinates & flux
        data_list = []
        for i in range(1, 4):
            data_list.append({
                'ra': 111.11 + i,
                'decl': 11.11 + i,
                'ra_fit_err': 0.01,
                'decl_fit_err': 0.01,
                'ra_sys_err': 20,
                'decl_sys_err': 20,
                'i_peak': 10 * i ,
                'i_peak_err': 0.1
            #  x=0.11, y=0.22, z=0.33, det_sigma=11.1, zone=i
            })
        # Insert the 3 sources in each image, while further varying the flux
        for i, image in enumerate(images):
            # Create the "source finding results"
            # Note that we reuse 'i_peak' as both peak & integrated flux.
            sources = []
            for data in data_list:
                source = (data['ra'], data['decl'],
                     data['ra_fit_err'], data['decl_fit_err'], # Gaussian fit errors
                     data['i_peak'] * (1 + i), data['i_peak_err'], # Peak
                     data['i_peak'] * (1 + i), data['i_peak_err'], # Integrated
                     10., # Significance level
                     1, 1, 0, # Beam params (width arcsec major, width arcsec minor, parallactic angle)
                     data['ra_sys_err'], data['decl_sys_err']) # Systematic errors
                sources.append(source)

            # Insert the sources
            image.insert_extracted_sources(sources)

            # Run the association for each list of source for an image
            image.associate_extracted_sources(deRuiter_r=3.7)

        # updates the dataset and its set of images
        self.dataset.update()
        self.dataset.update_images()

        # update the images and their sets of sources
        for image in self.dataset.images:
            image.update()
            image.update_sources()
            # Now pick any image, select the first source (smallest RA)

        # and extract its light curve
        sources = self.dataset.images.pop().sources
        sources = sorted(sources, key=attrgetter('ra'))
        lightcurve = sources[0].lightcurve()

        # check if the sources are associated in all images
        self.assertEqual(len(images), len(lightcurve))
        self.assertEqual(lightcurve[0][0], datetime.datetime(2010, 3, 3, 0, 0))
        self.assertEqual(lightcurve[1][0], datetime.datetime(2010, 3, 4, 0, 0))
        self.assertEqual(lightcurve[2][0], datetime.datetime(2010, 3, 5, 0, 0))
        self.assertEqual(lightcurve[3][0], datetime.datetime(2010, 3, 6, 0, 0))
        self.assertAlmostEqual(lightcurve[0][2], 10.)
        self.assertAlmostEqual(lightcurve[1][2], 20.)
        self.assertAlmostEqual(lightcurve[2][2], 30.)
        self.assertAlmostEqual(lightcurve[3][2], 40.)

        # Since the light curves are very similar, only eta_nu is different
        results = dbtransients._select_updated_variability_indices(self.dataset.images[-1].id)
        results = sorted(results, key=itemgetter('eta_int'))
        for result, eta_nu in zip(results, (16666.66666667, 66666.666666667,
                                            150000.0)):
            self.assertEqual(result['f_datapoints'], 4)
            self.assertAlmostEqual(result['eta_int'], eta_nu)
            self.assertAlmostEqual(result['v_int'], 0.516397779494)

<commit_msg>fix spacing<commit_after>from operator import attrgetter, itemgetter

import unittest2 as unittest

import datetime
from tkp.testutil.decorators import requires_database
from tkp.db.orm import DataSet
from tkp.db.orm import Image
import tkp.db
import tkp.db.transients as dbtransients

class TestLightCurve(unittest.TestCase):
    def setUp(self):
        self.database = tkp.db.Database()
        self.dataset = DataSet(data={'description': 'dataset with images'},
                               database=self.database)

    def tearDown(self):
        tkp.db.rollback()

    @requires_database()
    def test_lightcurve(self):
        # make 4 * 5 images with different date
        images = []
        for day in [3, 4, 5, 6]:
            data = {'taustart_ts': datetime.datetime(2010, 3, day),
                    'tau_time': 3600,
                    'url': '/',
                    'freq_eff': 80e6,
                    'freq_bw': 1e6,
                    'beam_smaj_pix': float(2.7),
                    'beam_smin_pix': float(2.3),
                    'beam_pa_rad': float(1.7),
                    'deltax': float(-0.01111),
                    'deltay': float(0.01111),
                    'centre_ra': 111,
                    'centre_decl': 11,
                    'xtr_radius' : 3
            }
            image = Image(dataset=self.dataset, data=data)
            images.append(image)

        # 3 sources per image, with different coordinates & flux
        data_list = []
        for i in range(1, 4):
            data_list.append({
                'ra': 111.11 + i,
                'decl': 11.11 + i,
                'ra_fit_err': 0.01,
                'decl_fit_err': 0.01,
                'ra_sys_err': 20,
                'decl_sys_err': 20,
                'i_peak': 10 * i ,
                'i_peak_err': 0.1
            #  x=0.11, y=0.22, z=0.33, det_sigma=11.1, zone=i
            })
        # Insert the 3 sources in each image, while further varying the flux
        for i, image in enumerate(images):
            # Create the "source finding results"
            # Note that we reuse 'i_peak' as both peak & integrated flux.
            sources = []
            for data in data_list:
                source = (data['ra'], data['decl'],
                     data['ra_fit_err'], data['decl_fit_err'],  # Gaussian fit errors
                     data['i_peak'] * (1 + i), data['i_peak_err'],  # Peak
                     data['i_peak'] * (1 + i), data['i_peak_err'],  # Integrated
                     10.,  # Significance level
                     1, 1,  0, # Beam params (width arcsec major, width arcsec minor, parallactic angle)
                     data['ra_sys_err'], data['decl_sys_err'])  # Systematic errors
                sources.append(source)

            # Insert the sources
            image.insert_extracted_sources(sources)

            # Run the association for each list of source for an image
            image.associate_extracted_sources(deRuiter_r=3.7)

        # updates the dataset and its set of images
        self.dataset.update()
        self.dataset.update_images()

        # update the images and their sets of sources
        for image in self.dataset.images:
            image.update()
            image.update_sources()
            # Now pick any image, select the first source (smallest RA)

        # and extract its light curve
        sources = self.dataset.images.pop().sources
        sources = sorted(sources, key=attrgetter('ra'))
        lightcurve = sources[0].lightcurve()

        # check if the sources are associated in all images
        self.assertEqual(len(images), len(lightcurve))
        self.assertEqual(lightcurve[0][0], datetime.datetime(2010, 3, 3, 0, 0))
        self.assertEqual(lightcurve[1][0], datetime.datetime(2010, 3, 4, 0, 0))
        self.assertEqual(lightcurve[2][0], datetime.datetime(2010, 3, 5, 0, 0))
        self.assertEqual(lightcurve[3][0], datetime.datetime(2010, 3, 6, 0, 0))
        self.assertAlmostEqual(lightcurve[0][2], 10.)
        self.assertAlmostEqual(lightcurve[1][2], 20.)
        self.assertAlmostEqual(lightcurve[2][2], 30.)
        self.assertAlmostEqual(lightcurve[3][2], 40.)

        # Since the light curves are very similar, only eta_nu is different
        results = dbtransients._select_updated_variability_indices(self.dataset.images[-1].id)
        results = sorted(results, key=itemgetter('eta_int'))
        for result, eta_nu in zip(results, (16666.66666667, 66666.666666667,
                                            150000.0)):
            self.assertEqual(result['f_datapoints'], 4)
            self.assertAlmostEqual(result['eta_int'], eta_nu)
            self.assertAlmostEqual(result['v_int'], 0.516397779494)

<|endoftext|>


<commit_before>##
#	\namespace	blur3d.api.classes.filesequence
#
#	\remarks	This file defines the FileSequence class that allows to manipulate a file sequence.
#
#	\author		douglas@blur.com
#	\author		Blur Studio
#	\date		09/11/11
#

#------------------------------------------------------------------------------------------------------------------------

import os

from framerange import FrameRange

#------------------------------------------------------------------------------------------------------------------------

class FileSequence( object ):

	def __init__( self, path, step=1 ):
		"""
			\remarks	Initialize the class.
		"""
		self._path = path
		self._step = step

	def path( self ):
		return os.path.abspath( self._path )

	def step( self ):
		return self._step

	def name( self ):
		return os.path.split( self._path )[1]

	def nameMask( self ):
		return '%(baseName)s.%(start)s-%(end)s.%(extension)s'

	def nameTokens( self ):
		import re
		regex = re.compile(r'^((?P<extension>[A-Za-z][A-Za-z][A-Za-z]).)((?P<range>(?P<end>[0-9]+)\-(?P<start>[0-9]+)).)(?P<baseName>[A-Za-z0-9_.]+)$')
		match = regex.match( self.name()[::-1] )
		if match:
			dict =  match.groupdict()
			for key in dict.keys():
				dict[ key ] = dict[ key ][::-1]
			return dict
		return None

	def nameToken( self, key ):
		if self.nameTokens().has_key( key ):
			return self.nameTokens()[ key ]
		else:
			return None

	def baseName( self ):
		return self.nameToken( 'baseName' )

	def uniqueName( self, rangePlaceHolder='' ):
		nameSplit = [ self.baseName(), self.extension() ]
		if rangePlaceHolder:
			nameSplit.insert( 1, rangePlaceHolder )
		return '.'.join( nameSplit )

	def frameRange( self, returnsAsString=False ):
		if returnsAsString:
			return self.nameToken( 'range' )
		return FrameRange( [ self.start(), self.end() ] )

	def start( self ):
		return int( self.nameToken( 'start' ) )

	def end( self ):
		return int( self.nameToken( 'end' ) )

	def extension( self ):
		return self.nameToken( 'extension' )

	def count( self ):
		return self.end() - self.start() + 1

	def setRange( self, range ):
		tokens = self.nameTokens()
		tokens[ 'start' ] = str( range[0] )
		tokens[ 'end' ] = str( range[1] )
		fileName = self.nameMask() % tokens
		self._path = os.path.join( self.basePath(), fileName )

	def padding( self ):
		padding = 1
		start = self.nameToken( 'start' )
		if not start == '0':
			for digit in start:
				if digit == '0':
					padding = padding + 1
		return padding

	def basePath( self ):
		return os.path.split( self._path )[0]

	def paddingCode( self ):
		return '%' + str( self.padding() ) + 'd'

	def codeName( self ):
		return self.baseName() + '.' + self.paddingCode() + '.' + self.extension()

	def codePath( self ):
		return os.path.join( self.basePath(), self.codeName() )

	def uniquePath( self, rangePlaceHolder='' ):
		return os.path.join( self.basePath(), self.uniqueName( rangePlaceHolder ) )

	def exists( self ):
		import glob
		paths = glob.glob( os.path.join( self.basePath(), self.baseName() + '.*.' + self.extension() ) )
		if len( paths ) > 0:
			return True
		return False

	def paths( self ):
		paths = []
		for frame in range( self.start(), self.end() + 1, self._step ):
			name = self.baseName() + '.' + str( frame ).zfill( self.padding() ) + '.' + self.extension()
			paths.append( os.path.join( self.basePath(), name ) )
		return paths

	def isComplete( self ):
		if len( self.missingFrames() ) > 0:
			return False
		return True

	def missingFrames( self ):
		frames = []
		for frame in range( self.start(), self.end() + 1, self._step ):
			name = self.baseName() + '.' + str( frame ).zfill( self.padding() ) + '.' + self.extension()
			path = os.path.join( self.basePath(), name )
			if not os.path.exists( path ):
				frames.append( frame )
		return frames

	def copy( self, output ):
		if self.isComplete():
			if self.count() == output.count():
				import shutil
				for i, o in zip( self.paths(), output.paths() ):
					shutil.copy( i, o )
				return True
			return False

	def delete( self, deletesBasePath=False ):
		if deletesBasePath:
			basePath = self.basePath()
			if os.path.exists( basePath ):
				os.rmdir( basePath )
		else:
			for path in self.paths():
				if os.path.exists( path ):
					os.remove( path )

	def generateMOV( self, outputPath=None, fps=30, ffmpeg='ffmpeg' ):
		if not outputPath:
			outputPath = os.path.join(( self.basePath() ), self.baseName() + '.mov' )
		extension = os.path.splitext( outputPath )[1]
		if extension == '.mov':
			if self.isComplete():
				normalisedSequencePath = os.path.join(  self.basePath(), self.baseName() + '_Temp.1-' + str( self.count() ) + '.' + self.extension() )
				normalisedSequence = FileSequence( normalisedSequencePath )
				self.copy( normalisedSequence )
				while not normalisedSequence.isComplete():
					continue
				import subprocess
				command = [ ffmpeg, '-r', str( fps ), "-i", normalisedSequence.codePath(), '-vcodec', 'mjpeg', '-qscale', '1', '-y', outputPath ]
				process = subprocess.Popen( command )
				process.communicate()
				normalisedSequence.delete()
				return True
			else:
				raise( 'Input sequence ' + str( self._path ) + ' is missing frames' )
		else:
			raise( 'Argument 1 should be a path to a mov file' )
		return True

	def link( self, output ):
		if self.isComplete():
			if input.count() == output.count():
				import subprocess
				for input, ouput in zip( input.paths(), output.paths() ):
					command = " ".join( [ "mklink", output, input ] )
					process = subprocess.Popen( command, shell = True )
				return True
		return False<commit_msg>Debugging someone.<commit_after>##
#	\namespace	blur3d.api.classes.filesequence
#
#	\remarks	This file defines the FileSequence class that allows to manipulate a file sequence.
#
#	\author		douglas@blur.com
#	\author		Blur Studio
#	\date		09/11/11
#

#------------------------------------------------------------------------------------------------------------------------

import os

from framerange import FrameRange

#------------------------------------------------------------------------------------------------------------------------

class FileSequence( object ):

	def __init__( self, path, step=1 ):
		"""
			\remarks	Initialize the class.
		"""
		self._path = path
		self._step = step

	def path( self ):
		return os.path.abspath( self._path )

	def step( self ):
		return self._step

	def name( self ):
		return os.path.split( self._path )[1]

	def nameMask( self ):
		return '%(baseName)s.%(start)s-%(end)s.%(extension)s'

	def nameTokens( self ):
		import re
		regex = re.compile(r'^((?P<extension>[A-Za-z][A-Za-z][A-Za-z]).)((?P<range>(?P<end>[0-9]+)\-(?P<start>[0-9]+)).)(?P<baseName>[A-Za-z0-9_.]+)$')
		match = regex.match( self.name()[::-1] )
		if match:
			dict =  match.groupdict()
			for key in dict.keys():
				dict[ key ] = dict[ key ][::-1]
			return dict
		return None

	def nameToken( self, key ):
		if self.nameTokens().has_key( key ):
			return self.nameTokens()[ key ]
		else:
			return None

	def baseName( self ):
		return self.nameToken( 'baseName' )

	def uniqueName( self, rangePlaceHolder='' ):
		nameSplit = [ self.baseName(), self.extension() ]
		if rangePlaceHolder:
			nameSplit.insert( 1, rangePlaceHolder )
		return '.'.join( nameSplit )

	def frameRange( self, returnsAsString=False ):
		if returnsAsString:
			return self.nameToken( 'range' )
		return FrameRange( [ self.start(), self.end() ] )

	def start( self ):
		return int( self.nameToken( 'start' ) )

	def end( self ):
		return int( self.nameToken( 'end' ) )

	def extension( self ):
		return self.nameToken( 'extension' )

	def count( self ):
		return self.end() - self.start() + 1

	def setRange( self, range ):
		tokens = self.nameTokens()
		print self.name()
		print self.path()
		tokens[ 'start' ] = str( range[0] )
		tokens[ 'end' ] = str( range[1] )
		fileName = self.nameMask() % tokens
		self._path = os.path.join( self.basePath(), fileName )

	def padding( self ):
		padding = 1
		start = self.nameToken( 'start' )
		if not start == '0':
			for digit in start:
				if digit == '0':
					padding = padding + 1
		return padding

	def basePath( self ):
		return os.path.split( self._path )[0]

	def paddingCode( self ):
		return '%' + str( self.padding() ) + 'd'

	def codeName( self ):
		return self.baseName() + '.' + self.paddingCode() + '.' + self.extension()

	def codePath( self ):
		return os.path.join( self.basePath(), self.codeName() )

	def uniquePath( self, rangePlaceHolder='' ):
		return os.path.join( self.basePath(), self.uniqueName( rangePlaceHolder ) )

	def exists( self ):
		import glob
		paths = glob.glob( os.path.join( self.basePath(), self.baseName() + '.*.' + self.extension() ) )
		if len( paths ) > 0:
			return True
		return False

	def paths( self ):
		paths = []
		for frame in range( self.start(), self.end() + 1, self._step ):
			name = self.baseName() + '.' + str( frame ).zfill( self.padding() ) + '.' + self.extension()
			paths.append( os.path.join( self.basePath(), name ) )
		return paths

	def isComplete( self ):
		if len( self.missingFrames() ) > 0:
			return False
		return True

	def missingFrames( self ):
		frames = []
		for frame in range( self.start(), self.end() + 1, self._step ):
			name = self.baseName() + '.' + str( frame ).zfill( self.padding() ) + '.' + self.extension()
			path = os.path.join( self.basePath(), name )
			if not os.path.exists( path ):
				frames.append( frame )
		return frames

	def copy( self, output ):
		if self.isComplete():
			if self.count() == output.count():
				import shutil
				for i, o in zip( self.paths(), output.paths() ):
					shutil.copy( i, o )
				return True
			return False

	def delete( self, deletesBasePath=False ):
		if deletesBasePath:
			basePath = self.basePath()
			if os.path.exists( basePath ):
				os.rmdir( basePath )
		else:
			for path in self.paths():
				if os.path.exists( path ):
					os.remove( path )

	def generateMOV( self, outputPath=None, fps=30, ffmpeg='ffmpeg' ):
		if not outputPath:
			outputPath = os.path.join(( self.basePath() ), self.baseName() + '.mov' )
		extension = os.path.splitext( outputPath )[1]
		if extension == '.mov':
			if self.isComplete():
				normalisedSequencePath = os.path.join(  self.basePath(), self.baseName() + '_Temp.1-' + str( self.count() ) + '.' + self.extension() )
				normalisedSequence = FileSequence( normalisedSequencePath )
				self.copy( normalisedSequence )
				while not normalisedSequence.isComplete():
					continue
				import subprocess
				command = [ ffmpeg, '-r', str( fps ), "-i", normalisedSequence.codePath(), '-vcodec', 'mjpeg', '-qscale', '1', '-y', outputPath ]
				process = subprocess.Popen( command )
				process.communicate()
				normalisedSequence.delete()
				return True
			else:
				raise( 'Input sequence ' + str( self._path ) + ' is missing frames' )
		else:
			raise( 'Argument 1 should be a path to a mov file' )
		return True

	def link( self, output ):
		if self.isComplete():
			if input.count() == output.count():
				import subprocess
				for input, ouput in zip( input.paths(), output.paths() ):
					command = " ".join( [ "mklink", output, input ] )
					process = subprocess.Popen( command, shell = True )
				return True
		return False<|endoftext|>


<commit_before># coding=UTF-8
from __future__ import absolute_import, division, print_function
import unittest
import struct
import io
import six

from coolamqp.framing.field_table import enframe_table, deframe_table, frame_table_size, \
                                         enframe_field_value, deframe_field_value, frame_field_value_size

if six.PY3:
    buffer = memoryview

class TestFramingTables(unittest.TestCase):
    def test_frame_unframe_table(self):

        tab = [
                (b'field', (b'yo', b's'))
            ]

        buf = io.BytesIO()

        self.assertEquals(frame_table_size(tab), 4+6+4)

        enframe_table(buf, tab)
        buf = buf.getvalue()

        self.assertEquals(buf, struct.pack('!I', 10) + b'\x05fields\x02yo')

        tab, delta = deframe_table(memoryview(buf), 0)

        self.assertEquals(len(tab), 1)
        self.assertEquals(delta, 14)
        self.assertEquals(tab[0], (b'field', (b'yo', b's')))

    def test_frame_unframe_value(self):

        buf = io.BytesIO()

        enframe_field_value(buf, (b'yo', b's'))

        buf = buf.getvalue()
        self.assertEquals(b's\x02yo', buf)

        fv, delta = deframe_field_value(memoryview(buf), 0)
        self.assertEquals(fv, (b'yo', b's'))
        self.assertEquals(delta, 4)

        self.assertEquals(frame_field_value_size(fv), 4)
<commit_msg>who needs tests<commit_after><|endoftext|>


<commit_before>#####################################################################################
#
#  Copyright (C) Tavendo GmbH
#
#  Unless a separate license agreement exists between you and Tavendo GmbH (e.g. you
#  have purchased a commercial license), the license terms below apply.
#
#  Should you enter into a separate license agreement after having received a copy of
#  this software, then the terms of such license agreement replace the terms below at
#  the time at which such license agreement becomes effective.
#
#  In case a separate license agreement ends, and such agreement ends without being
#  replaced by another separate license agreement, the license terms below apply
#  from the time at which said agreement ends.
#
#  LICENSE TERMS
#
#  This program is free software: you can redistribute it and/or modify it under the
#  terms of the GNU Affero General Public License, version 3, as published by the
#  Free Software Foundation. This program is distributed in the hope that it will be
#  useful, but WITHOUT ANY WARRANTY; without even the implied warranty of
#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
#
#  See the GNU Affero General Public License Version 3 for more details.
#
#  You should have received a copy of the GNU Affero General Public license along
#  with this program. If not, see <http://www.gnu.org/licenses/agpl-3.0.en.html>.
#
#####################################################################################

import datetime
import json
import hmac
import hashlib
import base64

from crossbar._logging import make_logger

from netaddr.ip import IPAddress, IPNetwork

from twisted.web.resource import Resource
from twisted.python.compat import nativeString

from autobahn.websocket.utf8validator import Utf8Validator
_validator = Utf8Validator()

class _InvalidUnicode(BaseException):
    """
    Invalid Unicode was found.
    """

class _CommonResource(Resource):
    """
    Shared components between PublisherResource and CallerResource.
    """
    isLeaf = True

    def __init__(self, options, session):
        """
        Ctor.

        :param options: Options for path service from configuration.
        :type options: dict
        :param session: Instance of `ApplicationSession` to be used for forwarding events.
        :type session: obj
        """
        Resource.__init__(self)
        self._options = options
        self._session = session
        self.log = make_logger()

        self._key = None
        if 'key' in options:
            self._key = options['key'].encode('utf8')

        self._secret = None
        if 'secret' in options:
            self._secret = options['secret'].encode('utf8')

        self._post_body_limit = int(options.get('post_body_limit', 0))
        self._timestamp_delta_limit = int(options.get('timestamp_delta_limit', 300))

        self._require_ip = None
        if 'require_ip' in options:
            self._require_ip = [IPNetwork(net) for net in options['require_ip']]

        self._require_tls = options.get('require_tls', None)

    def _deny_request(self, request, code, reason, **kwargs):
        """
        Called when client request is denied.
        """
        self.log.debug("[request denied] - {code} / " + reason,
                       code=code, **kwargs)
        request.setResponseCode(code)
        return reason.format(**kwargs).encode("utf8") + b"\n"

    def render(self, request):
        self.log.debug("[render] method={request.method} path={request.path} args={request.args}",
                       request=request)

        if request.method != b"POST":
            return self._deny_request(request, 405, u"HTTP/{0} not allowed".format(nativeString(request.method)))
        else:
            return self.render_POST(request)

    def render_POST(self, request):
        """
        Receives an HTTP/POST request, and then calls the Publisher/Caller
        processor.
        """
        # read HTTP/POST body
        body = request.content.read()

        args = request.args
        headers = request.getAllHeaders()

        # check content type + charset encoding
        #
        content_type_elements = [
            x.strip().lower()
            for x in headers.get(b"content-type", b'').split(b";")
        ]

        if b'application/json' != content_type_elements.pop(0):
            return self._deny_request(
                request, 400,
                u"bad or missing content type, should be 'application/json'")

        try:
            encoding_parts = {}

            for item in content_type_elements:
                # Parsing things like:
                # charset=utf-8
                _ = nativeString(item).split("=")
                assert len(_) == 2

                # We don't want duplicates
                key = _[0].strip().lower()
                assert key not in encoding_parts
                encoding_parts[key] = _[1].strip().lower()
        except:
            return self._deny_request(request, 400,
                                      u"mangled Content-Type header")

        charset_encoding = encoding_parts.get("charset", "utf-8")

        if charset_encoding not in ["utf-8", "utf8"]:
            return self._deny_request(
                request, 400,
                (u"'{charset_encoding}' is not an accepted charset encoding, "
                 u"must be utf-8"),
                charset_encoding=charset_encoding)

        # enforce "post_body_limit"
        #
        body_length = len(body)
        content_length = int(headers.get(b"content-length", body_length))

        if body_length != content_length:
            # Prevent the body length from being different to the given
            # Content-Length. This is so that clients can't lie and bypass
            # length restrictions by giving an incorrect header with a large
            # body.
            return self._deny_request(request, 400, u"HTTP/POST body length ({0}) is different to Content-Length ({1})".format(body_length, content_length))

        if self._post_body_limit and content_length > self._post_body_limit:
            return self._deny_request(request, 400, u"HTTP/POST body length ({0}) exceeds maximum ({1})".format(content_length, self._post_body_limit))

        #
        # parse/check HTTP/POST query parameters
        #

        # key
        #
        if 'key' in args:
            key_str = args["key"][0]
        else:
            if self._secret:
                return self._deny_request(request, 400, u"signed request required, but mandatory 'key' field missing")

        # timestamp
        #
        if 'timestamp' in args:
            timestamp_str = args["timestamp"][0]
            try:
                ts = datetime.datetime.strptime(nativeString(timestamp_str), "%Y-%m-%dT%H:%M:%S.%fZ")
                delta = abs((ts - datetime.datetime.utcnow()).total_seconds())
                if self._timestamp_delta_limit and delta > self._timestamp_delta_limit:
                    return self._deny_request(request, 400, u"request expired (delta {0} seconds)".format(delta))
            except ValueError:
                return self._deny_request(request, 400, u"invalid timestamp '{0}' (must be UTC/ISO-8601, e.g. '2011-10-14T16:59:51.123Z')".format(timestamp_str))
        else:
            if self._secret:
                return self._deny_request(request, 400, u"signed request required, but mandatory 'timestamp' field missing")

        # seq
        #
        if 'seq' in args:
            seq_str = args["seq"][0]
            try:
                # FIXME: check sequence
                seq = int(seq_str)  # noqa
            except:
                return self._deny_request(request, 400, u"invalid sequence number '{0}' (must be an integer)".format(seq_str))
        else:
            if self._secret:
                return self._deny_request(request, 400, u"signed request required, but mandatory 'seq' field missing")

        # nonce
        #
        if 'nonce' in args:
            nonce_str = args["nonce"][0]
            try:
                # FIXME: check nonce
                nonce = int(nonce_str)  # noqa
            except:
                return self._deny_request(request, 400, u"invalid nonce '{0}' (must be an integer)".format(nonce_str))
        else:
            if self._secret:
                return self._deny_request(request, 400, u"signed request required, but mandatory 'nonce' field missing")

        # signature
        #
        if 'signature' in args:
            signature_str = args["signature"][0]
        else:
            if self._secret:
                return self._deny_request(request, 400, u"signed request required, but mandatory 'signature' field missing")

        # do more checks if signed requests are required
        #
        if self._secret:

            if key_str != self._key:
                return self._deny_request(request, 400, u"unknown key '{0}' in signed request".format(nativeString(key_str)))

            # Compute signature: HMAC[SHA256]_{secret} (key | timestamp | seq | nonce | body) => signature
            hm = hmac.new(self._secret, None, hashlib.sha256)
            hm.update(key_str)
            hm.update(timestamp_str)
            hm.update(seq_str)
            hm.update(nonce_str)
            hm.update(body)
            signature_recomputed = base64.urlsafe_b64encode(hm.digest())

            if signature_str != signature_recomputed:
                return self._deny_request(request, 401, u"invalid request signature")
            else:
                self.log.debug("ok, request signature valid.")

        # user_agent = headers.get("user-agent", "unknown")
        client_ip = request.getClientIP()
        is_secure = request.isSecure()

        # enforce client IP address
        #
        if self._require_ip:
            ip = IPAddress(nativeString(client_ip))
            allowed = False
            for net in self._require_ip:
                if ip in net:
                    allowed = True
                    break
            if not allowed:
                return self._deny_request(request, 400, u"request denied based on IP address")

        # enforce TLS
        #
        if self._require_tls:
            if not is_secure:
                return self._deny_request(request, 400, u"request denied because not using TLS")

        # FIXME: authorize request
        authorized = True

        if not authorized:
            return self._deny_request(request, 401, u"not authorized")

        try:
            _validator.reset()
            validation_result = _validator.validate(body)

            # validate() returns a 4-tuple, of which item 0 is whether it
            # is valid
            if not validation_result[0]:
                raise _InvalidUnicode()
            event = json.loads(body.decode("utf8"))

        except (UnicodeDecodeError, _InvalidUnicode):
            return self._deny_request(
                request, 400,
                u"invalid request event - HTTP/POST body was invalid UTF-8")
        except Exception as e:
            print(e)
            return self._deny_request(
                request, 400,
                (u"invalid request event - HTTP/POST body must be valid "
                 u"JSON: {exc}"), exc=e)

        if not isinstance(event, dict):
            return self._deny_request(
                request, 400,
                u"invalid request event - HTTP/POST body must be JSON dict")

        return self._process(request, event)

    def _process(self, request, event):
        raise NotImplementedError()
<commit_msg>remove print statement<commit_after>#####################################################################################
#
#  Copyright (C) Tavendo GmbH
#
#  Unless a separate license agreement exists between you and Tavendo GmbH (e.g. you
#  have purchased a commercial license), the license terms below apply.
#
#  Should you enter into a separate license agreement after having received a copy of
#  this software, then the terms of such license agreement replace the terms below at
#  the time at which such license agreement becomes effective.
#
#  In case a separate license agreement ends, and such agreement ends without being
#  replaced by another separate license agreement, the license terms below apply
#  from the time at which said agreement ends.
#
#  LICENSE TERMS
#
#  This program is free software: you can redistribute it and/or modify it under the
#  terms of the GNU Affero General Public License, version 3, as published by the
#  Free Software Foundation. This program is distributed in the hope that it will be
#  useful, but WITHOUT ANY WARRANTY; without even the implied warranty of
#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
#
#  See the GNU Affero General Public License Version 3 for more details.
#
#  You should have received a copy of the GNU Affero General Public license along
#  with this program. If not, see <http://www.gnu.org/licenses/agpl-3.0.en.html>.
#
#####################################################################################

import datetime
import json
import hmac
import hashlib
import base64

from crossbar._logging import make_logger

from netaddr.ip import IPAddress, IPNetwork

from twisted.web.resource import Resource
from twisted.python.compat import nativeString

from autobahn.websocket.utf8validator import Utf8Validator
_validator = Utf8Validator()

class _InvalidUnicode(BaseException):
    """
    Invalid Unicode was found.
    """

class _CommonResource(Resource):
    """
    Shared components between PublisherResource and CallerResource.
    """
    isLeaf = True

    def __init__(self, options, session):
        """
        Ctor.

        :param options: Options for path service from configuration.
        :type options: dict
        :param session: Instance of `ApplicationSession` to be used for forwarding events.
        :type session: obj
        """
        Resource.__init__(self)
        self._options = options
        self._session = session
        self.log = make_logger()

        self._key = None
        if 'key' in options:
            self._key = options['key'].encode('utf8')

        self._secret = None
        if 'secret' in options:
            self._secret = options['secret'].encode('utf8')

        self._post_body_limit = int(options.get('post_body_limit', 0))
        self._timestamp_delta_limit = int(options.get('timestamp_delta_limit', 300))

        self._require_ip = None
        if 'require_ip' in options:
            self._require_ip = [IPNetwork(net) for net in options['require_ip']]

        self._require_tls = options.get('require_tls', None)

    def _deny_request(self, request, code, reason, **kwargs):
        """
        Called when client request is denied.
        """
        self.log.debug("[request denied] - {code} / " + reason,
                       code=code, **kwargs)
        request.setResponseCode(code)
        return reason.format(**kwargs).encode("utf8") + b"\n"

    def render(self, request):
        self.log.debug("[render] method={request.method} path={request.path} args={request.args}",
                       request=request)

        if request.method != b"POST":
            return self._deny_request(request, 405, u"HTTP/{0} not allowed".format(nativeString(request.method)))
        else:
            return self.render_POST(request)

    def render_POST(self, request):
        """
        Receives an HTTP/POST request, and then calls the Publisher/Caller
        processor.
        """
        # read HTTP/POST body
        body = request.content.read()

        args = request.args
        headers = request.getAllHeaders()

        # check content type + charset encoding
        #
        content_type_elements = [
            x.strip().lower()
            for x in headers.get(b"content-type", b'').split(b";")
        ]

        if b'application/json' != content_type_elements.pop(0):
            return self._deny_request(
                request, 400,
                u"bad or missing content type, should be 'application/json'")

        try:
            encoding_parts = {}

            for item in content_type_elements:
                # Parsing things like:
                # charset=utf-8
                _ = nativeString(item).split("=")
                assert len(_) == 2

                # We don't want duplicates
                key = _[0].strip().lower()
                assert key not in encoding_parts
                encoding_parts[key] = _[1].strip().lower()
        except:
            return self._deny_request(request, 400,
                                      u"mangled Content-Type header")

        charset_encoding = encoding_parts.get("charset", "utf-8")

        if charset_encoding not in ["utf-8", "utf8"]:
            return self._deny_request(
                request, 400,
                (u"'{charset_encoding}' is not an accepted charset encoding, "
                 u"must be utf-8"),
                charset_encoding=charset_encoding)

        # enforce "post_body_limit"
        #
        body_length = len(body)
        content_length = int(headers.get(b"content-length", body_length))

        if body_length != content_length:
            # Prevent the body length from being different to the given
            # Content-Length. This is so that clients can't lie and bypass
            # length restrictions by giving an incorrect header with a large
            # body.
            return self._deny_request(request, 400, u"HTTP/POST body length ({0}) is different to Content-Length ({1})".format(body_length, content_length))

        if self._post_body_limit and content_length > self._post_body_limit:
            return self._deny_request(request, 400, u"HTTP/POST body length ({0}) exceeds maximum ({1})".format(content_length, self._post_body_limit))

        #
        # parse/check HTTP/POST query parameters
        #

        # key
        #
        if 'key' in args:
            key_str = args["key"][0]
        else:
            if self._secret:
                return self._deny_request(request, 400, u"signed request required, but mandatory 'key' field missing")

        # timestamp
        #
        if 'timestamp' in args:
            timestamp_str = args["timestamp"][0]
            try:
                ts = datetime.datetime.strptime(nativeString(timestamp_str), "%Y-%m-%dT%H:%M:%S.%fZ")
                delta = abs((ts - datetime.datetime.utcnow()).total_seconds())
                if self._timestamp_delta_limit and delta > self._timestamp_delta_limit:
                    return self._deny_request(request, 400, u"request expired (delta {0} seconds)".format(delta))
            except ValueError:
                return self._deny_request(request, 400, u"invalid timestamp '{0}' (must be UTC/ISO-8601, e.g. '2011-10-14T16:59:51.123Z')".format(timestamp_str))
        else:
            if self._secret:
                return self._deny_request(request, 400, u"signed request required, but mandatory 'timestamp' field missing")

        # seq
        #
        if 'seq' in args:
            seq_str = args["seq"][0]
            try:
                # FIXME: check sequence
                seq = int(seq_str)  # noqa
            except:
                return self._deny_request(request, 400, u"invalid sequence number '{0}' (must be an integer)".format(seq_str))
        else:
            if self._secret:
                return self._deny_request(request, 400, u"signed request required, but mandatory 'seq' field missing")

        # nonce
        #
        if 'nonce' in args:
            nonce_str = args["nonce"][0]
            try:
                # FIXME: check nonce
                nonce = int(nonce_str)  # noqa
            except:
                return self._deny_request(request, 400, u"invalid nonce '{0}' (must be an integer)".format(nonce_str))
        else:
            if self._secret:
                return self._deny_request(request, 400, u"signed request required, but mandatory 'nonce' field missing")

        # signature
        #
        if 'signature' in args:
            signature_str = args["signature"][0]
        else:
            if self._secret:
                return self._deny_request(request, 400, u"signed request required, but mandatory 'signature' field missing")

        # do more checks if signed requests are required
        #
        if self._secret:

            if key_str != self._key:
                return self._deny_request(request, 400, u"unknown key '{0}' in signed request".format(nativeString(key_str)))

            # Compute signature: HMAC[SHA256]_{secret} (key | timestamp | seq | nonce | body) => signature
            hm = hmac.new(self._secret, None, hashlib.sha256)
            hm.update(key_str)
            hm.update(timestamp_str)
            hm.update(seq_str)
            hm.update(nonce_str)
            hm.update(body)
            signature_recomputed = base64.urlsafe_b64encode(hm.digest())

            if signature_str != signature_recomputed:
                return self._deny_request(request, 401, u"invalid request signature")
            else:
                self.log.debug("ok, request signature valid.")

        # user_agent = headers.get("user-agent", "unknown")
        client_ip = request.getClientIP()
        is_secure = request.isSecure()

        # enforce client IP address
        #
        if self._require_ip:
            ip = IPAddress(nativeString(client_ip))
            allowed = False
            for net in self._require_ip:
                if ip in net:
                    allowed = True
                    break
            if not allowed:
                return self._deny_request(request, 400, u"request denied based on IP address")

        # enforce TLS
        #
        if self._require_tls:
            if not is_secure:
                return self._deny_request(request, 400, u"request denied because not using TLS")

        # FIXME: authorize request
        authorized = True

        if not authorized:
            return self._deny_request(request, 401, u"not authorized")

        try:
            _validator.reset()
            validation_result = _validator.validate(body)

            # validate() returns a 4-tuple, of which item 0 is whether it
            # is valid
            if not validation_result[0]:
                raise _InvalidUnicode()
            event = json.loads(body.decode("utf8"))

        except (UnicodeDecodeError, _InvalidUnicode):
            return self._deny_request(
                request, 400,
                u"invalid request event - HTTP/POST body was invalid UTF-8")
        except Exception as e:
            return self._deny_request(
                request, 400,
                (u"invalid request event - HTTP/POST body must be valid "
                 u"JSON: {exc}"), exc=e)

        if not isinstance(event, dict):
            return self._deny_request(
                request, 400,
                u"invalid request event - HTTP/POST body must be JSON dict")

        return self._process(request, event)

    def _process(self, request, event):
        raise NotImplementedError()
<|endoftext|>


<commit_before>from datetime import timedelta, date
from django.db import models
from django.contrib.auth.models import User

class TipoProyecto(models.Model):
    nombre = models.CharField(max_length=140)
    impuesto = models.IntegerField(null=True)

    def __unicode__(self):
        return self.nombre

class Proyecto(models.Model):
    titulo = models.CharField(max_length=250, unique=True)
    descripcion = models.TextField()
    creador = models.ForeignKey(User, related_name="creador")
    contribuyentes = models.ManyToManyField(User, related_name="contribuyentes", blank=True, null=True)
    fecha_creacion = models.DateField(auto_now_add=True)
    fecha_modificacion = models.DateField(auto_now=True, auto_now_add=True)
    terminado = models.BooleanField(default=False)
    video_url = models.URLField(max_length=500)
    duracion = models.IntegerField()
    tipo_proyecto = models.ForeignKey(TipoProyecto)

    def __unicode__(self):
        return self.titulo + " " + str(self.terminado)

    def get_short_description(self):
        nombre_creador = self.creador.first_name + " " + self.creador.last_name
        return self.titulo + ", por " + nombre_creador

    def get_proyecto(self):
        nombre_creador = self.creador.username

        obj = {
            'id': self.id,
            'titulo': self.titulo,
            'creador': nombre_creador,
            'fecha_creacion': self.fecha_creacion,
            'duracion': self.duracion
        }

        return obj

    def get_nombre_creador(self):
        nombre_creador = self.creador.username
        return nombre_creador

    def get_dias_restantes(self):
        fecha_final = self.fecha_creacion + timedelta(days = self.duracion)
        dias_restantes = fecha_final - date.today()

        return dias_restantes

class ImagenProyecto(models.Model):
    imagen = models.FileField(upload_to="proyectos/thumbnails")
    proyecto = models.ForeignKey(Proyecto,null=True)
    fecha_creacion = models.DateTimeField(auto_now_add=True)

    def __unicode__(self):
        if self.proyecto is None:
            return " Sin proyecto " + str(self.fecha_creacion)

        return self.proyecto.titulo + " " + str(self.fecha_creacion)

class Moneda(models.Model):
    nombre = models.CharField(max_length=140)

    def __unicode__(self):
        return self.nombre

class Producto(models.Model):
    nombre = models.CharField(max_length=140)
    url = models.URLField(max_length=500)
    precio = models.IntegerField(default=0)
    moneda = models.ForeignKey(Moneda)
    es_recomendado = models.BooleanField(default=False)
    descripcion = models.TextField()
    proyecto = models.ForeignKey(Proyecto)

    def __unicode__(self):
        return self.nombre + " " + self.url

    def get_producto(self):
        obj = {
            'id': self.id,
            'nombre': self.nombre,
            'url': self.url,
            'precio': self.precio,
            'es_recomendado': self.es_recomendado
        }

        return obj

class ImagenProducto(models.Model):
    imagen = models.FileField(upload_to="productos/thumbnails")
    producto = models.ForeignKey(Producto,null=True)
    fecha_creacion = models.DateTimeField(auto_now_add=True)

    def __unicode__(self):
        return self.imagen + " " + self.fecha_creacion

class Categoria(models.Model):
    nombre = models.CharField(max_length=140)

    def __unicode__(self):
        return self.nombre

class Region(models.Model):
    nombre = models.CharField(max_length=140)

    def __unicode__(self):
        return self.nombre

class Comuna(models.Model):
    nombre = models.CharField(max_length=140)
    codigo = models.IntegerField()
    region = models.ForeignKey(Region)

    def __unicode__(self):
        return self.nombre + ", " + self.region.nombre

class TipoCuenta(models.Model):
    nombre = models.CharField(max_length=140)

    def __unicode__(self):
        return self.nombre

class Banco(models.Model):
    nombre = models.CharField(max_length=140)

    def __unicode__(self):
        return self.nombre

class CuentaBancaria(models.Model):
    numero_cuenta = models.IntegerField()
    tipo_cuenta = models.ForeignKey(TipoCuenta)
    banco = models.ForeignKey(Banco)

    def __unicode__(self):
        return self.numero_cuenta + " " + self.tipo_cuenta.nombre

class DetalleUsuario(models.Model):
    usuario = models.OneToOneField(User,related_name="usuario")
    fb_id = models.IntegerField(null=True, blank=True)
    rut = models.CharField(max_length=40, unique=True, null=True, blank=True)
    comuna = models.ForeignKey(Comuna, null=True, blank=True)
    sexo = models.BooleanField(default=True)
    cuenta_bancaria = models.ForeignKey(CuentaBancaria, null=True, blank=True)
    fecha_nacimiento = models.DateField(null=True, blank=True)
    url_img = models.URLField(max_length=1000)

    def __unicode__(self):
        return self.usuario.username
<commit_msg>se actualiza modelo<commit_after>from datetime import timedelta, date
from django.db import models
from django.contrib.auth.models import User

class TipoProyecto(models.Model):
    nombre = models.CharField(max_length=140)
    impuesto = models.IntegerField(null=True)

    def __unicode__(self):
        return self.nombre

class Proyecto(models.Model):
    titulo = models.CharField(max_length=250, unique=True)
    descripcion = models.TextField()
    creador = models.ForeignKey(User, related_name="creador")
    contribuyentes = models.ManyToManyField(User, related_name="contribuyentes", blank=True, null=True)
    fecha_creacion = models.DateField(auto_now_add=True)
    fecha_modificacion = models.DateField(auto_now=True, auto_now_add=True)
    terminado = models.BooleanField(default=False)
    video_url = models.URLField(max_length=500)
    duracion = models.IntegerField()
    tipo_proyecto = models.ForeignKey(TipoProyecto)

    def __unicode__(self):
        return self.titulo + " " + str(self.terminado)

    def get_short_description(self):
        nombre_creador = self.creador.first_name + " " + self.creador.last_name
        return self.titulo + ", por " + nombre_creador

    def get_proyecto(self):
        nombre_creador = self.creador.username

        obj = {
            'id': self.id,
            'titulo': self.titulo,
            'creador': nombre_creador,
            'fecha_creacion': self.fecha_creacion,
            'duracion': self.duracion
        }

        return obj

    def get_nombre_creador(self):
        nombre_creador = self.creador.username
        return nombre_creador

    def get_dias_restantes(self):
        fecha_final = self.fecha_creacion + timedelta(days = self.duracion)
        dias_restantes = fecha_final - date.today()

        return dias_restantes

class ImagenProyecto(models.Model):
    imagen = models.FileField(upload_to="proyectos/thumbnails")
    proyecto = models.ForeignKey(Proyecto,null=True)
    fecha_creacion = models.DateTimeField(auto_now_add=True)

    def __unicode__(self):
        if self.proyecto is None:
            return " Sin proyecto " + str(self.fecha_creacion)

        return self.proyecto.titulo + " " + str(self.fecha_creacion)

class Moneda(models.Model):
    nombre = models.CharField(max_length=140)

    def __unicode__(self):
        return self.nombre

class Producto(models.Model):
    nombre = models.CharField(max_length=140)
    url = models.URLField(max_length=500)
    precio = models.IntegerField(default=0)
    moneda = models.ForeignKey(Moneda)
    es_recomendado = models.BooleanField(default=False)
    descripcion = models.TextField()
    proyecto = models.ForeignKey(Proyecto)

    def __unicode__(self):
        return self.nombre + " " + self.url

    def get_producto(self):
        obj = {
            'id': self.id,
            'nombre': self.nombre,
            'url': self.url,
            'precio': self.precio,
            'es_recomendado': self.es_recomendado
        }

        return obj

class ImagenProducto(models.Model):
    imagen = models.FileField(upload_to="productos/thumbnails")
    producto = models.ForeignKey(Producto,null=True)
    fecha_creacion = models.DateTimeField(auto_now_add=True)

    def __unicode__(self):
        return self.imagen + " " + self.fecha_creacion

class Categoria(models.Model):
    nombre = models.CharField(max_length=140)

    def __unicode__(self):
        return self.nombre

class Region(models.Model):
    nombre = models.CharField(max_length=140)

    def __unicode__(self):
        return self.nombre

class Comuna(models.Model):
    nombre = models.CharField(max_length=140)
    codigo = models.IntegerField()
    region = models.ForeignKey(Region)

    def __unicode__(self):
        return self.nombre + ", " + self.region.nombre

class TipoCuenta(models.Model):
    nombre = models.CharField(max_length=140)

    def __unicode__(self):
        return self.nombre

class Banco(models.Model):
    nombre = models.CharField(max_length=140)

    def __unicode__(self):
        return self.nombre

class CuentaBancaria(models.Model):
    numero_cuenta = models.IntegerField()
    tipo_cuenta = models.ForeignKey(TipoCuenta)
    banco = models.ForeignKey(Banco)

    def __unicode__(self):
        return self.numero_cuenta + " " + self.tipo_cuenta.nombre

class DetalleUsuario(models.Model):
    usuario = models.OneToOneField(User, related_name="usuario")
    fb_id = models.IntegerField(null=True, blank=True)
    rut = models.CharField(max_length=40, unique=True, null=True, blank=True)
    comuna = models.ForeignKey(Comuna, null=True, blank=True)
    sexo = models.BooleanField(null=True, blank=True)
    cuenta_bancaria = models.ForeignKey(CuentaBancaria, null=True, blank=True)
    fecha_nacimiento = models.DateField(null=True, blank=True)
    avatar = models.URLField(max_length=1000,null=True)

    def __unicode__(self):
        return self.usuario.username
<|endoftext|>


<commit_before>from scripttest import TestFileEnvironment
from filecmp import cmp

testdir = "tests/testenv_graphprot_classification/"

env = TestFileEnvironment(testdir)

def test_invocation_no_params():
    "Call without parameters should return usage information."
    call = "../../GraphProt.pl"
    run = env.run(
        call,
        expect_error=True)
    assert run.returncode == 2

def test_eden_accuracy1():
    "Train a model."
    call = "../../GraphProt.pl --action train --fasta ../testedenacc.train.positives.fa --negfasta ../testedenacc.train.negatives.fa -prefix test_edenacc_train --keep-tmp"
    env.run(call)

def test_eden_accuracy2():
    "Now test on train."
    call = "../../GraphProt.pl --action predict --fasta ../testedenacc.train.positives.fa --negfasta ../testedenacc.train.negatives.fa -model test_edenacc_train.model -prefix test_edenacc_manualtestontrain --keep-tmp"
    run = env.run(call)
    assert "test_edenacc_manualtestontrain.predictions" in run.files_created
    cmp("tests/test_edenacc_manualtestontrain.predictions", testdir + "test_edenacc_manualtestontrain.predictions")

def test_classification_cv():
    "Test crossvalidation with classifiation."
    call = """../../GraphProt.pl -mode classification -action cv \
              -fasta ../testclip.train.positives.fa \
              -negfasta ../testclip.train.negatives.fa \
              -abstraction 1 -R 0 -D 0 -epochs 2 -lambda 0.1 -bitsize 10 -prefix CL_cv --keep-tmp"""
    env.run(call)
    cmp("tests/CL_cv.cv_results", testdir + "CL_cv.cv_results")

def test_classification_train():
    "Test model training with classification."
    call = """../../GraphProt.pl -mode classification -action train \
              -fasta ../testclip.train.positives.fa \
              -negfasta ../testclip.train.negatives.fa \
              -abstraction 1 -R 0 -D 0 -epochs 2 -lambda 0.1 -bitsize 10 \
              -prefix CL_train --keep-tmp"""
    env.run(call)
    cmp("tests/CL_train.model", testdir + "CL_train.model")

def test_classification_predict():
    "Test predicion with classification."
    call = """../../GraphProt.pl -mode classification -action predict \
              -fasta ../testclip.train.positives.fa \
              -negfasta ../testclip.train.negatives.fa \
              -model CL_train.model -abstraction 1 -R 0 -D 0 \
              -epochs 2 -lambda 0.1 -bitsize 10 -prefix CL_predict --keep-tmp"""
    env.run(call)
    cmp("tests/CL_predict.predictions", testdir + "CL_predict.predictions")

def test_classification_predict_only_positives():
    "Test prediction using only positives."
    call = """../../GraphProt.pl -mode classification -action predict \
              -fasta ../testclip.train.positives.fa \
              -model CL_train.model \
              -abstraction 1 -R 0 -D 0 -epochs 2 -lambda 0.1 -bitsize 10 \
              -prefix CL_predict_onlypos --keep-tmp"""
    env.run(call)
    cmp("tests/CL_predict_onlypos.predictions", testdir + "CL_predict_onlypos.predictions")

def test_classification_predict_ntmargins():
    "Test prediction of nucleotide-wise margins."
    call = """../../GraphProt.pl -mode classification -action predict_profile \
              --onlyseq \
              -fasta ../testclip.train.positives.fa \
              -model CL_train.model \
              -abstraction 1 -R 0 -D 0 -epochs 2 -lambda 0.1 -bitsize 10 \
              -prefix CL_ntmargins --keep-tmp"""
    env.run(call)
    cmp("tests/CL_ntmargins.profile", testdir + "CL_ntmargins.profile")

def test_classification_predict_has():
    "Test prediction of high affinity sites."
    call = """../../GraphProt.pl -mode classification -action predict_has --onlyseq \
              -fasta ../testclip.train.positives.fa \
              -model CL_train.model \
              -abstraction 1 -R 0 -D 0 -epochs 2 -lambda 0.1 -bitsize 10 -percentile 55 \
              -prefix CL_has --keep-tmp"""
    env.run(call)
    cmp("tests/CL_has.has", testdir + "CL_has.has")

def test_classification_motif():
    "Test motif calculation."
    call = """../../GraphProt.pl -mode classification -action motif \
              -fasta ../testclip.train.positives.fa -model CL_train.model \
              -abstraction 1 -R 0 -D 0 -epochs 2 -lambda 0.1 -bitsize 10 \
              -prefix CL_motif --keep-tmp"""
    run = env.run(call)
    assert "CL_motif.structure_motif.png" in run.files_created
    assert "CL_motif.structure_motif_pairedunpaired.png" in run.files_created
    assert "CL_motif.sequence_motif.png" in run.files_created
    cmp("tests/CL_motif.sequence_motif", testdir + "CL_motif.sequence_motif")
    cmp("tests/CL_motif.structure_motif", testdir + "CL_motif.structure_motif")

def test_classification_motif_onlyseq():
    "Test motif calculation."
    call = """../../GraphProt.pl -mode classification -action motif \
              --onlyseq \
              -fasta ../testclip.train.positives.fa -model CL_train.model \
              -R 1 -D 0 -bitsize 10 -prefix CL_onlyseq --keep-tmp"""
    run = env.run(call)
    assert "CL_onlyseq.sequence_motif.png" in run.files_created
    cmp("tests/CL_onlyseq.sequence_motif", testdir + "CL_onlyseq.sequence_motif")

def test_classification_profile_all_vp():
    "Calculate profiles for pure viewpoint sequences."
    call = """../../GraphProt.pl -mode classification -action predict_profile \
              -fasta ../testclip.twoseqs300nt_allcaps.fa \
              -model CL_train.model \
              -abstraction 1 -R 0 -D 0 -epochs 2 -lambda 0.1 -bitsize 10 \
              -prefix testclip.twoseqs300nt_allcaps.margins --keep-tmp"""
    env.run(call)
    cmp("tests/testclip.twoseqs300nt_allcaps.margins.profile",
        testdir + "testclip.twoseqs300nt_allcaps.margins.profile")

def test_classification_profile_center_vp():
    "Calculate profiles for center viewpoint seqeunces."
    call = """../../GraphProt.pl -mode classification -action predict_profile \
              -fasta ../testclip.twoseqs300nt.fa -model CL_train.model \
              -abstraction 1 -R 0 -D 0 -epochs 2 -lambda 0.1 -bitsize 10 \
              -prefix testclip.twoseqs300nt.margins --keep-tmp"""
    env.run(call)
    cmp("tests/testclip.twoseqs300nt.margins.profile",
        testdir + "testclip.twoseqs300nt.margins.profile")

def test_classification_motif_center_vp():
    "Calculate motif for center viewpoint jsequences."
    call = "../../GraphProt.pl -mode classification -action motif \
            -fasta ../testclip.twoseqs300nt.fa -model CL_train.model \
            -abstraction 1 -R 0 -D 0 -epochs 2 -lambda 0.1 -bitsize 10 \
            -prefix testclip.twoseqs300nt.motif --keep-tmp"""
    run = env.run(call)
    assert "testclip.twoseqs300nt.motif.structure_motif.png" in run.files_created
    assert "testclip.twoseqs300nt.motif.structure_motif_pairedunpaired.png" in run.files_created
    assert "testclip.twoseqs300nt.motif.sequence_motif.png" in run.files_created
    cmp("tests/testclip.twoseqs300nt.motif.sequence_motif", testdir + "testclip.twoseqs300nt.motif.sequence_motif")
    cmp("tests/testclip.twoseqs300nt.motif.structure_motif", testdir + "testclip.twoseqs300nt.motif.structure_motif")
<commit_msg>make tests independent of each other<commit_after>from scripttest import TestFileEnvironment
from filecmp import cmp

testdir = "tests/testenv_graphprot_classification/"

env = TestFileEnvironment(testdir)

def test_invocation_no_params():
    "Call without parameters should return usage information."
    call = "../../GraphProt.pl"
    run = env.run(
        call,
        expect_error=True)
    assert run.returncode == 2

def test_eden_accuracy1():
    "Train a model."
    call = """../../GraphProt.pl --action train \
              --fasta ../testedenacc.train.positives.fa \
              --negfasta ../testedenacc.train.negatives.fa \
              -prefix test_edenacc_train --keep-tmp"""
    env.run(call)

def test_eden_accuracy2():
    "Now test on train."
    call = """../../GraphProt.pl --action predict \
              --fasta ../testedenacc.train.positives.fa \
              --negfasta ../testedenacc.train.negatives.fa \
              -model ../test_edenacc_train.model \
              -prefix test_edenacc_manualtestontrain --keep-tmp"""
    run = env.run(call)
    assert "test_edenacc_manualtestontrain.predictions" in run.files_created
    cmp("tests/test_edenacc_manualtestontrain.predictions", testdir + "test_edenacc_manualtestontrain.predictions")

def test_classification_cv():
    "Test crossvalidation with classifiation."
    call = """../../GraphProt.pl -mode classification -action cv \
              -fasta ../testclip.train.positives.fa \
              -negfasta ../testclip.train.negatives.fa \
              -abstraction 1 -R 0 -D 0 -epochs 2 -lambda 0.1 -bitsize 10 -prefix CL_cv --keep-tmp"""
    env.run(call)
    cmp("tests/CL_cv.cv_results", testdir + "CL_cv.cv_results")

def test_classification_train():
    "Test model training with classification."
    call = """../../GraphProt.pl -mode classification -action train \
              -fasta ../testclip.train.positives.fa \
              -negfasta ../testclip.train.negatives.fa \
              -abstraction 1 -R 0 -D 0 -epochs 2 -lambda 0.1 -bitsize 10 \
              -prefix CL_train --keep-tmp"""
    env.run(call)
    cmp("tests/CL_train.model", testdir + "CL_train.model")

def test_classification_predict():
    "Test predicion with classification."
    call = """../../GraphProt.pl -mode classification -action predict \
              -fasta ../testclip.train.positives.fa \
              -negfasta ../testclip.train.negatives.fa \
              -model ../CL_train.model -abstraction 1 -R 0 -D 0 \
              -epochs 2 -lambda 0.1 -bitsize 10 -prefix CL_predict --keep-tmp"""
    env.run(call)
    cmp("tests/CL_predict.predictions", testdir + "CL_predict.predictions")

def test_classification_predict_only_positives():
    "Test prediction using only positives."
    call = """../../GraphProt.pl -mode classification -action predict \
              -fasta ../testclip.train.positives.fa \
              -model ../CL_train.model \
              -abstraction 1 -R 0 -D 0 -epochs 2 -lambda 0.1 -bitsize 10 \
              -prefix CL_predict_onlypos --keep-tmp"""
    env.run(call)
    cmp("tests/CL_predict_onlypos.predictions", testdir + "CL_predict_onlypos.predictions")

def test_classification_predict_ntmargins():
    "Test prediction of nucleotide-wise margins."
    call = """../../GraphProt.pl -mode classification -action predict_profile \
              --onlyseq \
              -fasta ../testclip.train.positives.fa \
              -model ../CL_train.model \
              -abstraction 1 -R 0 -D 0 -epochs 2 -lambda 0.1 -bitsize 10 \
              -prefix CL_ntmargins --keep-tmp"""
    env.run(call)
    cmp("tests/CL_ntmargins.profile", testdir + "CL_ntmargins.profile")

def test_classification_predict_has():
    "Test prediction of high affinity sites."
    call = """../../GraphProt.pl -mode classification -action predict_has --onlyseq \
              -fasta ../testclip.train.positives.fa \
              -model ../CL_train.model \
              -abstraction 1 -R 0 -D 0 -epochs 2 -lambda 0.1 -bitsize 10 -percentile 55 \
              -prefix CL_has --keep-tmp"""
    env.run(call)
    cmp("tests/CL_has.has", testdir + "CL_has.has")

def test_classification_motif():
    "Test motif calculation."
    call = """../../GraphProt.pl -mode classification -action motif \
              -fasta ../testclip.train.positives.fa \
              -model ../CL_train.model \
              -abstraction 1 -R 0 -D 0 -epochs 2 -lambda 0.1 -bitsize 10 \
              -prefix CL_motif --keep-tmp"""
    run = env.run(call)
    assert "CL_motif.structure_motif.png" in run.files_created
    assert "CL_motif.structure_motif_pairedunpaired.png" in run.files_created
    assert "CL_motif.sequence_motif.png" in run.files_created
    cmp("tests/CL_motif.sequence_motif", testdir + "CL_motif.sequence_motif")
    cmp("tests/CL_motif.structure_motif", testdir + "CL_motif.structure_motif")

def test_classification_motif_onlyseq():
    "Test motif calculation."
    call = """../../GraphProt.pl -mode classification -action motif \
              --onlyseq \
              -fasta ../testclip.train.positives.fa \
              -model ../CL_train.model \
              -R 1 -D 0 -bitsize 10 -prefix CL_onlyseq --keep-tmp"""
    run = env.run(call)
    assert "CL_onlyseq.sequence_motif.png" in run.files_created
    cmp("tests/CL_onlyseq.sequence_motif", testdir + "CL_onlyseq.sequence_motif")

def test_classification_profile_all_vp():
    "Calculate profiles for pure viewpoint sequences."
    call = """../../GraphProt.pl -mode classification -action predict_profile \
              -fasta ../testclip.twoseqs300nt_allcaps.fa \
              -model ../CL_train.model \
              -abstraction 1 -R 0 -D 0 -epochs 2 -lambda 0.1 -bitsize 10 \
              -prefix testclip.twoseqs300nt_allcaps.margins --keep-tmp"""
    env.run(call)
    cmp("tests/testclip.twoseqs300nt_allcaps.margins.profile",
        testdir + "testclip.twoseqs300nt_allcaps.margins.profile")

def test_classification_profile_center_vp():
    "Calculate profiles for center viewpoint seqeunces."
    call = """../../GraphProt.pl -mode classification -action predict_profile \
              -fasta ../testclip.twoseqs300nt.fa \
              -model ../CL_train.model \
              -abstraction 1 -R 0 -D 0 -epochs 2 -lambda 0.1 -bitsize 10 \
              -prefix testclip.twoseqs300nt.margins --keep-tmp"""
    env.run(call)
    cmp("tests/testclip.twoseqs300nt.margins.profile",
        testdir + "testclip.twoseqs300nt.margins.profile")

def test_classification_motif_center_vp():
    "Calculate motif for center viewpoint jsequences."
    call = "../../GraphProt.pl -mode classification -action motif \
            -fasta ../testclip.twoseqs300nt.fa \
            -model ../CL_train.model \
            -abstraction 1 -R 0 -D 0 -epochs 2 -lambda 0.1 -bitsize 10 \
            -prefix testclip.twoseqs300nt.motif --keep-tmp"""
    run = env.run(call)
    assert "testclip.twoseqs300nt.motif.structure_motif.png" in run.files_created
    assert "testclip.twoseqs300nt.motif.structure_motif_pairedunpaired.png" in run.files_created
    assert "testclip.twoseqs300nt.motif.sequence_motif.png" in run.files_created
    cmp("tests/testclip.twoseqs300nt.motif.sequence_motif", testdir + "testclip.twoseqs300nt.motif.sequence_motif")
    cmp("tests/testclip.twoseqs300nt.motif.structure_motif", testdir + "testclip.twoseqs300nt.motif.structure_motif")
<|endoftext|>