<commit_before># -*- coding: utf-8 -*-
"""
Created on Tue Jul 19 10:18:36 2016

@author: SLG
"""
import keras
from keras.models import Sequential
from keras.layers.core import Dense, Dropout, Activation, Flatten
from keras.layers.convolutional import Convolution2D, MaxPooling2D
from keras.optimizers import SGD
import cPickle
import math
import numpy as np
from keras.regularizers import l1l2
from keras.layers.normalization import BatchNormalization
from keras.callbacks import EarlyStopping
from scipy.stats import ttest_ind

class LossHistory(keras.callbacks.Callback):
    def on_train_begin(self, logs={}):
        self.losses = []

    def on_batch_end(self, batch, logs={}):
        self.losses.append(logs.get('loss'))

#Hyperparameters for tuning
weight_init = ['he_normal','glorot_normal']
regl1 = [1.0, 0.1, 0.01, 0.001, 0.0]
regl2 = [1.0, 0.1, 0.01, 0.001, 0.0]
dropout = [0.0, 0.25, 0.5, 0.7]
bsize = [32, 70, 100, 150]
learning_rate = [0.0001, 0.0003, 0.001, 0.003, 0.01, 0.03, 0.1, 0.3, 1, 3]
optimizer = ['adadelta', 'sgd']

n_dataset = 5

def get_data(n_dataset):
    f = file('MODS_dataset_cv_{0}.pkl'.format(n_dataset),'rb')
    data = cPickle.load(f)
    f.close()
    training_data = data[0]
    validation_data = data[1]
    t_data = training_data[0]
    t_label = training_data[1]
    v_data = validation_data[0]
    v_label = validation_data[1]

    #reshape to 256 x 192; 1: greyscale
    t_data = t_data.reshape(t_data.shape[0], 1, 256, 192)
    v_data = v_data.reshape(v_data.shape[0], 1, 256, 192)

    #less precision means less memory needed: 64 -> 32 (half the memory used)
    t_data = t_data.astype('float32')
    v_data = v_data.astype('float32')

    return t_data, t_label, v_data, v_label

def network(regl1, regl2, weight_init, dropout, optimize):

    #create network architecture
    model = Sequential()

    model.add(Convolution2D(16, 5, 5,input_shape=(1, 256, 192),W_regularizer=l1l2(l1=regl1, l2=regl2),init=weight_init))
    model.add(BatchNormalization())
    model.add(Activation('relu'))
    model.add(MaxPooling2D(pool_size=(2, 2)))
    model.add(Dropout(dropout))

    model.add(Convolution2D(32, 3, 3, W_regularizer=l1l2(l1=regl1, l2=regl2),init=weight_init))
    model.add(BatchNormalization())
    model.add(Activation('relu'))
    model.add(MaxPooling2D(pool_size=(2, 2)))
    model.add(Dropout(dropout))

    model.add(Convolution2D(64, 3, 3, W_regularizer=l1l2(l1=regl1, l2=regl2),init=weight_init))
    model.add(BatchNormalization())
    model.add(Activation('relu'))
    model.add(MaxPooling2D(pool_size=(2, 2)))
    model.add(Dropout(dropout))

    model.add(Convolution2D(64, 3, 3, W_regularizer=l1l2(l1=regl1, l2=regl2),init=weight_init))
    model.add(BatchNormalization())
    model.add(Activation('relu'))
    model.add(MaxPooling2D(pool_size=(2, 2)))
    model.add(Dropout(dropout))

    model.add(Convolution2D(64, 3, 3, W_regularizer=l1l2(l1=regl1, l2=regl2),init=weight_init))
    model.add(BatchNormalization())
    model.add(Activation('relu'))
    model.add(MaxPooling2D(pool_size=(2, 2)))
    model.add(Dropout(dropout))

    model.add(Flatten())
    model.add(Dense(50,W_regularizer=l1l2(l1=regl1, l2=regl2),init=weight_init))
    #model.add(BatchNormalization())
    model.add(Activation('relu'))
    model.add(Dropout(0.5))

    model.add(Dense(output_dim=1))
    model.add(Activation('sigmoid'))

    model.compile(loss='binary_crossentropy',class_mode='binary', optimizer=optimize)

    return model

def cv_calc(n_dataset, regl1, regl2, weight_init, dropout, optimize, bsize):

    #conditions for training, where early stopping stops the network training if validation accuracy does not increase
    early_stopping = EarlyStopping(monitor='val_loss', patience=6)
    history = LossHistory()

    #creates a dictionary of parameters and their values
    dict_param = {'Regularization L1': regl1, 'Regularization L2': regl2, 'Weight initialization': weight_init,
                  'Dropout': dropout, 'Optimizer': optimize, 'Mini batch size': bsize}

    #creates a list of metrics according to dataset
    param_metrics = []

    #for each dataset, loads the model, the data, and calculates metrics
    for i in xrange(n_dataset):
        X_train, Y_train, X_val, Y_val = get_data(i)
        model = network(regl1, regl2, weight_init, dropout, optimize)
        train_datagen = keras.preprocessing.image.ImageDataGenerator(
            rotation_range=20,
            horizontal_flip=True,
            vertical_flip=True,
            )
        model.fit_generator(train_datagen.flow(X_train, Y_train, batch_size=bsize),
                    samples_per_epoch=len(X_train), nb_epoch=10, callbacks = [early_stopping, history])

        prediction = model.predict(X_val,batch_size=Y_val.shape[0])

        prediction[prediction > 0.5] = 1.0
        prediction[prediction <= 0.5] = 0.0

        F1 = 0.0
        mcc = 0.0

        TP = 0
        for i in xrange(len(Y_val)):
            if Y_val[i] == 1.0 and prediction[i] == 1.0:
			TP+=1.0

        TN = 0
        for i in xrange(len(Y_val)):
            if Y_val[i]  == 0.0 and prediction[i] == 0.0:
                TN+= 1.0

        FP = 0
        for i in xrange(len(Y_val)):
            if Y_val[i] < prediction[i]:
                FP+= 1.0

        FN = 0
        for i in xrange(len(Y_val)):
            if Y_val[i] > prediction[i]:
                FN+= 1.0

        print (TP, TN, FP, FN)

        sensitivity = TP/(TP + FN)
        specificity = TN/(TN + FP)

        print (sensitivity, specificity)

        print('Test Sensitivity Score: {0:.2%}'.format(sensitivity))
        print('Test Specificity Score: {0:.2%}'.format(specificity))

        try:
            PPV = TP / (TP + FP)
            NPV = TN / (TN + FN)
            F1 = 2.0 * (PPV * sensitivity)/(PPV + sensitivity)
            mcc = (TP*TN - FP*FN)/(math.sqrt((TP + FP)*(TP + FN)*(TN + FP)*(TN + FN)))
            print('Test F1 Score: {0:.2}'.format(F1))

        except ZeroDivisionError:
            print('Divide by Zero')
            F1 = None
            mcc = None

        param_metrics.append([sensitivity, specificity, F1, PPV, NPV, mcc])
        score = model.evaluate(X_val, Y_val, show_accuracy=True, verbose=0)
        print('Test loss:', score[0])
        print('Test accuracy:', score[1])

        param_metrics = np.array(param_metrics)
        stat = [dict_param, param_metrics]

        return stat, model

def compare_data(best_models,stat,model):
    sens = stat[1][0,:]
    for m in xrange(len(best_models)):
        sens_best = best_models[m][1]
        None, p_value = ttest_ind(sens,sens_best)
        if p_value < 0.05:
            best_models[m] = [stat[0],sens,model]
            break
    return best_models
best_models = []

for i in weight_init:
    for j in regl1:
        for k in regl2:
            for l in dropout:
                for m in bsize:
                    for n in optimizer:
                        if optimizer=='sgd':
                            for o in learning_rate:
                                sgd = SGD(lr=o, decay=1e-6, momentum=0.9, nesterov=True)
                                stat, model = cv_calc(n_dataset, regl1 = j, regl2 = k , weight_init = i, dropout = l, optimize = sgd, bsize = m)

                                json_string = model.to_json()

	#Save weights
	name = 'MODS_keras_{0}_weights_cnn_{0}.h5'.format(n_dataset)
	model.save_weights(name,overwrite=True)

	#l = h5py.File("loss_history_{0}.hdf5".format(number_db), "w")
	#dset = f.create_dataset("loss_history_{0}".format(number_db), (100,), dtype='i')

	#A way to open a model with weights in the same arquitecture
	'''
	json_string = model.to_json()
	open('my_model_architecture.json', 'w').write(json_string)
	model.save_weights('my_model_weights.h5')
	'''
	import cPickle
	f = open('cnn_loss_{0}.pkl'.format(number_db),'wb')
	cPickle.dump(history.losses,f,protocol=cPickle.HIGHEST_PROTOCOL)
	f.close()

	h = open('cnn_metrics_{0}.pkl'.format(number_db),'wb')
	p = [sensitivity, specificity, F1, mcc]
	cPickle.dump(p ,h,protocol=cPickle.HIGHEST_PROTOCOL)
	h.close()
	model.reset_states()

    #returns best model

<commit_msg>Updating more stuff<commit_after># -*- coding: utf-8 -*-
"""
Created on Tue Jul 19 10:18:36 2016

@author: SLG
"""

import keras
from keras.models import Sequential
from keras.layers.core import Dense, Dropout, Activation, Flatten
from keras.layers.convolutional import Convolution2D, MaxPooling2D
from keras.optimizers import SGD
import cPickle
import math
import numpy as np
from keras.regularizers import l1l2
from keras.layers.normalization import BatchNormalization
from keras.callbacks import EarlyStopping
#from scipy.stats import ttest_ind

class LossHistory(keras.callbacks.Callback):
    def on_train_begin(self, logs={}):
        self.losses = []

    def on_batch_end(self, batch, logs={}):
        self.losses.append(logs.get('loss'))

#Hyperparameters for tuning
weight_init = ['he_normal','glorot_normal']
regl1 = [1.0, 0.1, 0.01, 0.001, 0.0]
regl2 = [1.0, 0.1, 0.01, 0.001, 0.0]
dropout = [0.0, 0.25, 0.5, 0.7]
bsize = [32, 70, 100, 150]
learning_rate = [0.0001, 0.0003, 0.001, 0.003, 0.01, 0.03, 0.1, 0.3, 1, 3]
optimizer = ['adadelta', 'sgd']

n_dataset = 5

def get_data(n_dataset):
    f = file('MODS_dataset_cv_{0}.pkl'.format(n_dataset),'rb')
    data = cPickle.load(f)
    f.close()
    training_data = data[0]
    validation_data = data[1]
    t_data = training_data[0]
    t_label = training_data[1]
    v_data = validation_data[0]
    v_label = validation_data[1]

    #reshape to 256 x 192; 1: greyscale
    t_data = t_data.reshape(t_data.shape[0], 1, 256, 192)
    v_data = v_data.reshape(v_data.shape[0], 1, 256, 192)

    #less precision means less memory needed: 64 -> 32 (half the memory used)
    t_data = t_data.astype('float32')
    v_data = v_data.astype('float32')

    return t_data, t_label, v_data, v_label

def network(regl1, regl2, weight_init, dropout, optimize):

    #create network architecture
    model = Sequential()

    model.add(Convolution2D(16, 5, 5,input_shape=(1, 256, 192),W_regularizer=l1l2(l1=regl1, l2=regl2),init=weight_init))
    model.add(BatchNormalization())
    model.add(Activation('relu'))
    model.add(MaxPooling2D(pool_size=(2, 2)))
    model.add(Dropout(dropout))

    model.add(Convolution2D(32, 3, 3, W_regularizer=l1l2(l1=regl1, l2=regl2),init=weight_init))
    model.add(BatchNormalization())
    model.add(Activation('relu'))
    model.add(MaxPooling2D(pool_size=(2, 2)))
    model.add(Dropout(dropout))

    model.add(Convolution2D(64, 3, 3, W_regularizer=l1l2(l1=regl1, l2=regl2),init=weight_init))
    model.add(BatchNormalization())
    model.add(Activation('relu'))
    model.add(MaxPooling2D(pool_size=(2, 2)))
    model.add(Dropout(dropout))

    model.add(Convolution2D(64, 3, 3, W_regularizer=l1l2(l1=regl1, l2=regl2),init=weight_init))
    model.add(BatchNormalization())
    model.add(Activation('relu'))
    model.add(MaxPooling2D(pool_size=(2, 2)))
    model.add(Dropout(dropout))

    model.add(Convolution2D(64, 3, 3, W_regularizer=l1l2(l1=regl1, l2=regl2),init=weight_init))
    model.add(BatchNormalization())
    model.add(Activation('relu'))
    model.add(MaxPooling2D(pool_size=(2, 2)))
    model.add(Dropout(dropout))

    model.add(Flatten())
    model.add(Dense(50,W_regularizer=l1l2(l1=regl1, l2=regl2),init=weight_init))
    #model.add(BatchNormalization())
    model.add(Activation('relu'))
    model.add(Dropout(0.5))

    model.add(Dense(output_dim=1))
    model.add(Activation('sigmoid'))

    model.compile(loss='binary_crossentropy',class_mode='binary', optimizer=optimize)

    return model

def cv_calc(n_dataset, regl1, regl2, weight_init, dropout, optimize, bsize):

    #conditions for training, where early stopping stops the network training if validation accuracy does not increase
    early_stopping = EarlyStopping(monitor='val_loss', patience=6)
    history = LossHistory()

    #creates a dictionary of parameters and their values
    dict_param = {'Regularization L1': regl1, 'Regularization L2': regl2, 'Weight initialization': weight_init,
                  'Dropout': dropout, 'Optimizer': optimize, 'Mini batch size': bsize}

    #creates a list of metrics according to dataset
    param_metrics = []

    #for each dataset, loads the model, the data, and calculates metrics
    for i in xrange(n_dataset):
        X_train, Y_train, X_val, Y_val = get_data(i)
        model = network(regl1, regl2, weight_init, dropout, optimize)
        train_datagen = keras.preprocessing.image.ImageDataGenerator(
            rotation_range=20,
            horizontal_flip=True,
            vertical_flip=True,
            )
        model.fit_generator(train_datagen.flow(X_train, Y_train, batch_size=bsize),
                    samples_per_epoch=len(X_train), nb_epoch=40, callbacks = [early_stopping, history])

        prediction = model.predict(X_val,batch_size=Y_val.shape[0])

        prediction[prediction > 0.5] = 1.0
        prediction[prediction <= 0.5] = 0.0

        F1 = 0.0
        mcc = 0.0

        TP = 0
        for i in xrange(len(Y_val)):
            if Y_val[i] == 1.0 and prediction[i] == 1.0:
			TP+=1.0

        TN = 0
        for i in xrange(len(Y_val)):
            if Y_val[i]  == 0.0 and prediction[i] == 0.0:
                TN+= 1.0

        FP = 0
        for i in xrange(len(Y_val)):
            if Y_val[i] < prediction[i]:
                FP+= 1.0

        FN = 0
        for i in xrange(len(Y_val)):
            if Y_val[i] > prediction[i]:
                FN+= 1.0

        print (TP, TN, FP, FN)

        sensitivity = TP/(TP + FN)
        specificity = TN/(TN + FP)

        print (sensitivity, specificity)

        print('Test Sensitivity Score: {0:.2%}'.format(sensitivity))
        print('Test Specificity Score: {0:.2%}'.format(specificity))

        try:
            PPV = TP / (TP + FP)
            NPV = TN / (TN + FN)
            F1 = 2.0 * (PPV * sensitivity)/(PPV + sensitivity)
            mcc = (TP*TN - FP*FN)/(math.sqrt((TP + FP)*(TP + FN)*(TN + FP)*(TN + FN)))
            print('Test F1 Score: {0:.2}'.format(F1))

        except ZeroDivisionError:
            print('Divide by Zero')
            F1 = None
            mcc = None

        param_metrics.append([sensitivity, specificity, F1, PPV, NPV, mcc])
        score = model.evaluate(X_val, Y_val, show_accuracy=True, verbose=0)
        print('Test loss:', score[0])
        print('Test accuracy:', score[1])

        param_metrics = np.array(param_metrics)
        stat = [dict_param, param_metrics]

        return stat, model

for i in weight_init:
    for j in regl1:
        for k in regl2:
            for l in dropout:
                for m in bsize:
                    for n in optimizer:
                        if optimizer=='sgd':
                            for o in learning_rate:
                                sgd = SGD(lr=o, decay=1e-6, momentum=0.9, nesterov=True)
                                stat, model = cv_calc(n_dataset, regl1 = j, regl2 = k , weight_init = i, dropout = l, optimize = sgd, bsize = m)

                                json_string = model.to_json()
                        else:
                            stat, model = cv_calc(n_dataset, regl1 = j, regl2 = k , weight_init = i, dropout = l, optimize = 'adadelta', bsize = m)
                            json_string = model.to_json()

	#Save weights
	name = 'MODS_keras_{0}_weights_cnn_{0}.h5'.format(n_dataset)
	model.save_weights(name,overwrite=True)

	#l = h5py.File("loss_history_{0}.hdf5".format(number_db), "w")
	#dset = f.create_dataset("loss_history_{0}".format(number_db), (100,), dtype='i')

	#A way to open a model with weights in the same arquitecture
	'''
	json_string = model.to_json()
	open('my_model_architecture.json', 'w').write(json_string)
	model.save_weights('my_model_weights.h5')
	'''
	f = open('cnn_loss_{0}.pkl'.format(n_dataset),'wb')
	cPickle.dump(cv_calc().history.losses,f,protocol=cPickle.HIGHEST_PROTOCOL)
	f.close()

	h = open('cnn_metrics_{0}.pkl'.format(n_dataset),'wb')
	cPickle.dump(stat,h,protocol=cPickle.HIGHEST_PROTOCOL)
	h.close()
	model.reset_states()

'''
##To compare models and return the best 5, according to sensitivity
def compare_data(best_models,stat,model):
    sens = stat[1][0,:]
    for m in xrange(len(best_models)):
        sens_best = best_models[m][1]
        None, p_value = ttest_ind(sens,sens_best)
        if p_value < 0.05:
            best_models[m] = [stat[0],sens,model]
            break
    return best_models
best_models = []
'''
<|endoftext|>


<commit_before>"""crashreport URL Configuration

The `urlpatterns` list routes URLs to views. For more information please see:
    https://docs.djangoproject.com/en/1.9/topics/http/urls/
Examples:
Function views
    1. Add an import:  from my_app import views
    2. Add a URL to urlpatterns:  url(r'^$', views.home, name='home')
Class-based views
    1. Add an import:  from other_app.views import Home
    2. Add a URL to urlpatterns:  url(r'^$', Home.as_view(), name='home')
Including another URLconf
    1. Add an import:  from blog import urls as blog_urls
    2. Import the include() function: from django.conf.urls import url, include
    3. Add a URL to urlpatterns:  url(r'^blog/', include(blog_urls)
"""

# -*- Mode: python; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
#

from django.conf.urls import include, url
from django.contrib import admin

urlpatterns = [
    url(r'^submit/', include('crashsubmit.urls')),
    url(r'^process/', include('processor.urls')),
    url(r'^symbols/', include('symbols.urls')),
    url(r'^upload/', include('symbols.urls')),
    url(r'^stats/', include('stats.urls')),
    url(r'^management/', include('management.urls')),
    url(r'^admin/', admin.site.urls),
    url(r'^accounts/', include('django.contrib.auth.urls')),
]

# vim:set shiftwidth=4 softtabstop=4 expandtab: */
<commit_msg>forward from the crashreport.libreoffice.org to the actual main page<commit_after>"""crashreport URL Configuration

The `urlpatterns` list routes URLs to views. For more information please see:
    https://docs.djangoproject.com/en/1.9/topics/http/urls/
Examples:
Function views
    1. Add an import:  from my_app import views
    2. Add a URL to urlpatterns:  url(r'^$', views.home, name='home')
Class-based views
    1. Add an import:  from other_app.views import Home
    2. Add a URL to urlpatterns:  url(r'^$', Home.as_view(), name='home')
Including another URLconf
    1. Add an import:  from blog import urls as blog_urls
    2. Import the include() function: from django.conf.urls import url, include
    3. Add a URL to urlpatterns:  url(r'^blog/', include(blog_urls)
"""

# -*- Mode: python; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
#

from django.conf.urls import include, url
from django.contrib import admin
from django.views.generic.base import RedirectView

urlpatterns = [
    url(r'^$', RedirectView.as_view(url='/stats/')),
    url(r'^submit/', include('crashsubmit.urls')),
    url(r'^process/', include('processor.urls')),
    url(r'^symbols/', include('symbols.urls')),
    url(r'^upload/', include('symbols.urls')),
    url(r'^stats/', include('stats.urls')),
    url(r'^management/', include('management.urls')),
    url(r'^admin/', admin.site.urls),
    url(r'^accounts/', include('django.contrib.auth.urls')),
]

# vim:set shiftwidth=4 softtabstop=4 expandtab: */
<|endoftext|>


<commit_before>from collections import namedtuple
from functools import partial
from heapq import heappush, heappop

edge_cost = namedtuple('edge_cost', 'id cost')

def add_edge(graph, left, right, cost):
    if left not in graph:
        graph[left] = []

    graph[left].append(edge_cost(id = right, cost = cost))

def make_graph(source):
    """
    >>> make_graph([(1, 2, 10), (2, 3, 15), (1, 3, 30)])
    {1: [edge_cost(id=2, cost=10), edge_cost(id=3, cost=30)], 2: [edge_cost(id=3, cost=15)]}
    """
    graph = {}

    for entry in source:
        add_edge(graph, *entry)

    return graph

def reverse_graph(source):
    """
    >>> reverse_graph(make_graph([(1, 2, 10), (2, 3, 15), (1, 3, 30)]))
    {2: [edge_cost(id=1, cost=10)], 3: [edge_cost(id=1, cost=30), edge_cost(id=2, cost=15)]}
    """
    reversed = {}

    for left, edges in source.items():
        for right, cost in edges:
            add_edge(reversed, right, left, cost)

    return reversed

def dijkstra_kernel(graph, start, end, previous):
    queue = [(0.0, start)]
    while queue:
        cost, id = heappop(queue)
        yield edge_cost(id = id, cost = cost)

        if id == end:
            return

        for neighbour in graph[id]:
            alt_cost = cost + neighbour.cost

            # if there was no cost associated or cost is lower
            if id not in previous or alt_cost < previous[neighbour.id].cost:
                heappush(queue, (alt_cost, neighbour.id))
                previous[neighbour.id] = edge_cost(id = id, cost = alt_cost)

def backtrack(previous, id):
    """
    >>> list(backtrack({3: edge_cost(id=2, cost=15), 2: edge_cost(id=1, cost=10)}, 3))
    [3, 2, 1]
    """
    yield id
    while id in previous:
        id = previous[id].id
        yield id

def dijkstra(graph, start, end):
    """
    >>> graph = make_graph([(1, 2, 10), (2, 3, 15), (1, 3, 30)])
    >>> dijkstra(graph, 1, 3)
    (25.0, [1, 2, 3])
    """
    previous = {}

    # keep visiting nodes till search is finished
    for id, cost in dijkstra_kernel(graph, start, end, previous):
        pass

    # if end node was reached
    if id == end:
        path = list(backtrack(previous, id))
        path.reverse()
        return (cost, path)

def bidirect_dijkstra(graph, start, end):
    """
    >>> graph = make_graph([(1, 2, 10), (2, 3, 15), (1, 3, 30)])
    >>> bidirect_dijkstra(graph, 1, 3)
    (25.0, [1, 2, 3])
    """

    fwd_previous = {}
    bwd_previous = {}

    fwd_kernel = dijkstra_kernel(graph, start, end, fwd_previous)
    bwd_kernel = dijkstra_kernel(reverse_graph(graph), end, start, bwd_previous)

    def check_intersects(id, previous, cost):
        if id in previous:
            alt_cost = fwd_previous[id].cost + bwd_previous[id].cost
            if alt_cost < cost:
                return (id, alt_cost)

    intersection = None
    cost = float('inf')

    for fwd, bwd in zip(fwd_kernel, bwd_kernel):
        # break if any search reached target
        #if fwd.id == end or bwd.id == start:
        #    break

        # check if there is better intersection
        intersection, cost = \
            check_intersects(fwd.id, bwd_previous, cost) or \
            check_intersects(bwd.id, fwd_previous, cost) or \
            (intersection, cost)

        # stop searching if current path is not improved
        if cost < fwd.cost + bwd.cost:
            break

    # if shortest path was found
    if intersection is not None:
        path = list(backtrack(fwd_previous, intersection))[1:]
        path.reverse()
        path.extend(backtrack(bwd_previous, intersection))

        return (cost, path)<commit_msg>Fixed comments<commit_after>from collections import namedtuple
from functools import partial
from heapq import heappush, heappop

edge_cost = namedtuple('edge_cost', 'id cost')

def add_edge(graph, left, right, cost):
    if left not in graph:
        graph[left] = []

    graph[left].append(edge_cost(id = right, cost = cost))

def make_graph(source):
    """
    Make graph out of (from, to, cost) tuples.

    Example:
    >>> make_graph([(1, 2, 10), (2, 3, 15), (1, 3, 30)])
    {1: [edge_cost(id=2, cost=10), edge_cost(id=3, cost=30)], 2: [edge_cost(id=3, cost=15)]}
    """
    graph = {}

    for entry in source:
        add_edge(graph, *entry)

    return graph

def reverse_graph(source):
    """
    Reverse direction of graph

    Example:
    >>> reverse_graph(make_graph([(1, 2, 10), (2, 3, 15), (1, 3, 30)]))
    {2: [edge_cost(id=1, cost=10)], 3: [edge_cost(id=1, cost=30), edge_cost(id=2, cost=15)]}
    """
    reversed = {}

    for left, edges in source.items():
        for right, cost in edges:
            add_edge(reversed, right, left, cost)

    return reversed

def dijkstra_kernel(graph, start, end, previous):
    queue = [(0.0, start)]
    while queue:
        cost, id = heappop(queue)
        yield edge_cost(id = id, cost = cost)

        # destination reached
        if id == end:
            return

        for neighbour in graph[id]:
            alt_cost = cost + neighbour.cost

            # if there was no cost associated or cost is lower
            if id not in previous or alt_cost < previous[neighbour.id].cost:
                heappush(queue, (alt_cost, neighbour.id))
                previous[neighbour.id] = edge_cost(id = id, cost = alt_cost)

def backtrack(previous, id):
    """
    >>> list(backtrack({3: edge_cost(id=2, cost=15), 2: edge_cost(id=1, cost=10)}, 3))
    [3, 2, 1]
    """
    yield id
    while id in previous:
        id = previous[id].id
        yield id

def dijkstra(graph, start, end):
    """
    >>> graph = make_graph([(1, 2, 10), (2, 3, 15), (1, 3, 30)])
    >>> dijkstra(graph, 1, 3)
    (25.0, [1, 2, 3])
    """
    previous = {}

    # keep visiting nodes till search is finished
    for id, cost in dijkstra_kernel(graph, start, end, previous):
        pass

    # if end node was reached
    if id == end:
        path = list(backtrack(previous, id))
        path.reverse()
        return (cost, path)

def bidirect_dijkstra(graph, start, end):
    """
    >>> graph = make_graph([(1, 2, 10), (2, 3, 15), (1, 3, 30)])
    >>> bidirect_dijkstra(graph, 1, 3)
    (25.0, [1, 2, 3])
    """

    fwd_previous = {}
    bwd_previous = {}

    fwd_kernel = dijkstra_kernel(graph, start, end, fwd_previous)
    bwd_kernel = dijkstra_kernel(reverse_graph(graph), end, start, bwd_previous)

    # helper for finding intersection candidate
    def check_intersects(id, previous, cost):
        if id in previous:
            alt_cost = fwd_previous[id].cost + bwd_previous[id].cost
            if alt_cost < cost:
                return (id, alt_cost)

    intersection = None
    cost = float('inf')

    for fwd, bwd in zip(fwd_kernel, bwd_kernel):
        # check if there is better intersection
        intersection, cost = \
            check_intersects(fwd.id, bwd_previous, cost) or \
            check_intersects(bwd.id, fwd_previous, cost) or \
            (intersection, cost)

        # stop searching if current path is not improved
        if cost < fwd.cost + bwd.cost:
            break

    # if shortest path was found
    if intersection is not None:
        path = list(backtrack(fwd_previous, intersection))[1:]
        path.reverse()
        path.extend(backtrack(bwd_previous, intersection))

        return (cost, path)<|endoftext|>


<commit_before>#!/usr/bin/env python3

import argparse
import os
import re
import shlex
import subprocess
import sys
from tempfile import NamedTemporaryFile

DESC = """

Determine whether `bindgen` can successfully process a C or C++ input header.

First, `bindgen` is run on the input header. Then the emitted bindings are
compiled with `rustc`. Finally, the compiled bindings' layout tests are run.

By default, this script will exit zero if all of the above steps are successful,
and non-zero if any of them fail. This is appropriate for determining if some
test case (perhaps generated with `csmith` or another fuzzer) uncovers any bugs
in `bindgen`.

However, this script can also be used when reducing (perhaps with `creduce`) a
known-bad test case into a new, smaller test case that exhibits the same bad
behavior. In this mode, you might expect that the emitted bindings fail to
compile with `rustc`, and want to exit non-zero early if that is not the
case. See the "reducing arguments" for details and what knobs are available.

"""

parser = argparse.ArgumentParser(
    formatter_class=argparse.RawDescriptionHelpFormatter,
    description=DESC.strip())

BINDGEN_ARGS = "--with-derive-partialeq \
--with-derive-eq \
--with-derive-partialord \
--with-derive-ord \
--with-derive-hash \
--with-derive-default"

parser.add_argument(
    "--bindgen-args",
    type=str,
    default=BINDGEN_ARGS,
    help="An argument string that `bindgen` should be invoked with. By default, all traits are derived. Note that the input header and output bindings file will automatically be provided by this script, and you should not manually specify them.")

parser.add_argument(
    "--save-temp-files",
    action="store_true",
    help="Do not delete temporary files.")

parser.add_argument(
    "input",
    type=str,
    default="input.h",
    help="The input header file. Defaults to 'input.h'.")

REDUCING_DESC = """

Arguments that are useful when reducing known-bad test cases into
equivalent-but-smaller test cases that exhibit the same bug with `creduce`.

"""

reducing = parser.add_argument_group("reducing arguments", REDUCING_DESC.strip())

reducing.add_argument(
    "--release",
    action="store_true",
    help="Use a release instead of a debug build.")
reducing.add_argument(
    "--expect-bindgen-fail",
    action="store_true",
    help="Exit non-zero if `bindgen` successfully emits bindings.")
reducing.add_argument(
    "--bindgen-grep",
    type=str,
    help="Exit non-zero if the given regexp pattern is not found in `bindgen`'s output.")
reducing.add_argument(
    "--bindings-grep",
    type=str,
    nargs='*',
    help="Exit non-zero if the given regexp pattern is not found in the emitted bindings.")

reducing.add_argument(
    "--no-compile-bindings",
    action="store_false",
    dest="rustc",
    help="Do not attempt to compile the emitted bindings with `rustc`.")
reducing.add_argument(
    "--extra-compile-file",
    type=str,
    help="Append the content of this extra file to the end of the emitted bindings just before compiling it.")
reducing.add_argument(
    "--expect-compile-fail",
    action="store_true",
    help="Exit non-zero if `rustc` successfully compiles the emitted bindings.")
reducing.add_argument(
    "--rustc-grep",
    type=str,
    help="Exit non-zero if the output from compiling the bindings with `rustc` does not contain the given regexp pattern")

reducing.add_argument(
    "--no-layout-tests",
    action="store_false",
    dest="layout_tests",
    help="Do not run the compiled bindings' layout tests.")
reducing.add_argument(
    "--expect-layout-tests-fail",
    action="store_true",
    help="Exit non-zero if the compiled bindings' layout tests pass.")
reducing.add_argument(
    "--layout-tests-grep",
    type=str,
    help="Exit non-zero if the output of running the compiled bindings' layout tests does not contain the given regexp pattern.")

################################################################################

class ExitOne(Exception):
    pass

def exit_1(msg, child=None):
    print(msg)

    if child:
        stdout = decode(child.stdout)
        for line in stdout.splitlines():
            sys.stdout.write("+")
            sys.stdout.write(line)
            sys.stdout.write("\n")
        stderr = decode(child.stderr)
        for line in stderr.splitlines():
            sys.stderr.write("+")
            sys.stderr.write(line)
            sys.stderr.write("\n")

    raise ExitOne()

def main():
    args = parser.parse_args()
    os.environ["RUST_BACKTRACE"] = "full"

    exit_code = 0
    try:
        bindings = new_temp_file(prefix="bindings-", suffix=".rs")
        run_bindgen(args, bindings)

        if args.rustc and not args.expect_bindgen_fail:
            test_exe = new_temp_file(prefix="layout-tests-")
            run_rustc(args, bindings, test_exe)

            if args.layout_tests and not args.expect_compile_fail:
                run_layout_tests(args, test_exe)
    except ExitOne:
        exit_code = 1
    except Exception as e:
        exit_code = 2
        print("Unexpected exception:", e)

    if not args.save_temp_files:
        for path in TEMP_FILES:
            try:
                os.remove(path)
            except Exception as e:
                print("Unexpected exception:", e)

    sys.exit(exit_code)

def run(cmd, **kwargs):
    print("Running:", cmd)
    return subprocess.run(cmd, **kwargs)

def decode(f):
    return f.decode(encoding="utf-8", errors="ignore")

TEMP_FILES = []

def new_temp_file(prefix, suffix=None):
    temp = NamedTemporaryFile(delete=False, prefix=prefix, suffix=suffix)
    temp.close()
    TEMP_FILES.append(temp.name)
    return temp.name

def contains(pattern, lines):
    for line in lines:
        if re.match(pattern, line):
            return True
    return False

def regexp(pattern):
    if not pattern.startswith("^"):
        pattern = ".*" + pattern
    if not pattern.endswith("$"):
        pattern = pattern + ".*"
    return re.compile(pattern)

def run_bindgen(args, bindings):
    manifest_path = os.path.abspath(os.path.join(os.path.dirname(sys.argv[0]),
                                                 "..",
                                                 "Cargo.toml"))
    command = ["cargo", "run", "--manifest-path", manifest_path]
    if args.release:
        command += ["--release"]
    command += ["--", args.input, "-o", bindings]
    command += shlex.split(args.bindgen_args)

    child = run(command, stderr=subprocess.PIPE, stdout=subprocess.PIPE)

    if args.bindgen_grep:
        pattern = regexp(args.bindgen_grep)
        if not (contains(pattern, decode(child.stdout).splitlines()) or
                contains(pattern, decode(child.stderr).splitlines())):
            exit_1("Error: did not find '{}' in `bindgen`'s output".format(args.bindgen_grep), child)

    if args.expect_bindgen_fail and child.returncode == 0:
        exit_1("Error: expected running `bindgen` to fail, but it didn't", child)

    if not args.expect_bindgen_fail and child.returncode != 0:
        exit_1("Error: running `bindgen` failed", child)

    for arg in args.bindings_grep:
        pattern = regexp(arg)
        with open(bindings, mode="r") as f:
            if not contains(pattern, f):
                print("Error: expected the emitted bindings to contain '{}', but they didn't".format(arg))
                print("---------- {} ----------------------------------------------".format(bindings))
                f.seek(0)
                print(f.read())
                raise ExitOne()

def run_rustc(args, bindings, test_exe):
    if args.extra_compile_file:
        with open(bindings, mode="a") as outfile:
            with open(args.extra_compile_file, mode="r") as infile:
                outfile.write(infile.read())
    child = run(
        ["rustc", "--crate-type", "lib", "--test", "-o", test_exe, bindings],
        stdout=subprocess.PIPE,
        stderr=subprocess.PIPE)

    if args.rustc_grep:
        pattern = regexp(args.rustc_grep)
        if not (contains(pattern, decode(child.stdout).splitlines()) or
                contains(pattern, decode(child.stderr).splitlines())):
            exit_1("Error: did not find '{}' in `rustc`'s output".format(args.rustc_grep), child)

    if args.expect_compile_fail and child.returncode == 0:
        exit_1("Error: expected running `rustc` on the emitted bindings to fail, but it didn't", child)

    if not args.expect_compile_fail and child.returncode != 0:
        exit_1("Error: running `rustc` on the emitted bindings failed", child)

def run_layout_tests(args, test_exe):
    child = run(
        [test_exe],
        stdout=subprocess.PIPE,
        stderr=subprocess.PIPE)

    if args.layout_tests_grep:
        pattern = regexp(args.layout_tests_grep)
        if not (contains(pattern, decode(child.stdout).splitlines()) or
                contains(pattern, decode(child.stderr).splitlines())):
            exit_1("Error: did not find '{}' in the compiled bindings' layout tests' output".format(args.layout_tests_grep), child)

    if args.expect_layout_tests_fail and child.returncode == 0:
        exit_1("Error: expected running the compiled bindings' layout tests to fail, but it didn't", child)

    if not args.expect_layout_tests_fail and child.returncode != 0:
        exit_1("Error: running the compiled bindings' layout tests failed", child)

if __name__ == "__main__":
    main()
<commit_msg>Fix NoneType error<commit_after>#!/usr/bin/env python3

import argparse
import os
import re
import shlex
import subprocess
import sys
from tempfile import NamedTemporaryFile

DESC = """

Determine whether `bindgen` can successfully process a C or C++ input header.

First, `bindgen` is run on the input header. Then the emitted bindings are
compiled with `rustc`. Finally, the compiled bindings' layout tests are run.

By default, this script will exit zero if all of the above steps are successful,
and non-zero if any of them fail. This is appropriate for determining if some
test case (perhaps generated with `csmith` or another fuzzer) uncovers any bugs
in `bindgen`.

However, this script can also be used when reducing (perhaps with `creduce`) a
known-bad test case into a new, smaller test case that exhibits the same bad
behavior. In this mode, you might expect that the emitted bindings fail to
compile with `rustc`, and want to exit non-zero early if that is not the
case. See the "reducing arguments" for details and what knobs are available.

"""

parser = argparse.ArgumentParser(
    formatter_class=argparse.RawDescriptionHelpFormatter,
    description=DESC.strip())

BINDGEN_ARGS = "--with-derive-partialeq \
--with-derive-eq \
--with-derive-partialord \
--with-derive-ord \
--with-derive-hash \
--with-derive-default"

parser.add_argument(
    "--bindgen-args",
    type=str,
    default=BINDGEN_ARGS,
    help="An argument string that `bindgen` should be invoked with. By default, all traits are derived. Note that the input header and output bindings file will automatically be provided by this script, and you should not manually specify them.")

parser.add_argument(
    "--save-temp-files",
    action="store_true",
    help="Do not delete temporary files.")

parser.add_argument(
    "input",
    type=str,
    default="input.h",
    help="The input header file. Defaults to 'input.h'.")

REDUCING_DESC = """

Arguments that are useful when reducing known-bad test cases into
equivalent-but-smaller test cases that exhibit the same bug with `creduce`.

"""

reducing = parser.add_argument_group("reducing arguments", REDUCING_DESC.strip())

reducing.add_argument(
    "--release",
    action="store_true",
    help="Use a release instead of a debug build.")
reducing.add_argument(
    "--expect-bindgen-fail",
    action="store_true",
    help="Exit non-zero if `bindgen` successfully emits bindings.")
reducing.add_argument(
    "--bindgen-grep",
    type=str,
    help="Exit non-zero if the given regexp pattern is not found in `bindgen`'s output.")
reducing.add_argument(
    "--bindings-grep",
    type=str,
    nargs='*',
    help="Exit non-zero if the given regexp pattern is not found in the emitted bindings.")

reducing.add_argument(
    "--no-compile-bindings",
    action="store_false",
    dest="rustc",
    help="Do not attempt to compile the emitted bindings with `rustc`.")
reducing.add_argument(
    "--extra-compile-file",
    type=str,
    help="Append the content of this extra file to the end of the emitted bindings just before compiling it.")
reducing.add_argument(
    "--expect-compile-fail",
    action="store_true",
    help="Exit non-zero if `rustc` successfully compiles the emitted bindings.")
reducing.add_argument(
    "--rustc-grep",
    type=str,
    help="Exit non-zero if the output from compiling the bindings with `rustc` does not contain the given regexp pattern")

reducing.add_argument(
    "--no-layout-tests",
    action="store_false",
    dest="layout_tests",
    help="Do not run the compiled bindings' layout tests.")
reducing.add_argument(
    "--expect-layout-tests-fail",
    action="store_true",
    help="Exit non-zero if the compiled bindings' layout tests pass.")
reducing.add_argument(
    "--layout-tests-grep",
    type=str,
    help="Exit non-zero if the output of running the compiled bindings' layout tests does not contain the given regexp pattern.")

################################################################################

class ExitOne(Exception):
    pass

def exit_1(msg, child=None):
    print(msg)

    if child:
        stdout = decode(child.stdout)
        for line in stdout.splitlines():
            sys.stdout.write("+")
            sys.stdout.write(line)
            sys.stdout.write("\n")
        stderr = decode(child.stderr)
        for line in stderr.splitlines():
            sys.stderr.write("+")
            sys.stderr.write(line)
            sys.stderr.write("\n")

    raise ExitOne()

def main():
    args = parser.parse_args()
    os.environ["RUST_BACKTRACE"] = "full"

    exit_code = 0
    try:
        bindings = new_temp_file(prefix="bindings-", suffix=".rs")
        run_bindgen(args, bindings)

        if args.rustc and not args.expect_bindgen_fail:
            test_exe = new_temp_file(prefix="layout-tests-")
            run_rustc(args, bindings, test_exe)

            if args.layout_tests and not args.expect_compile_fail:
                run_layout_tests(args, test_exe)
    except ExitOne:
        exit_code = 1
    except Exception as e:
        exit_code = 2
        print("Unexpected exception:", e)

    if not args.save_temp_files:
        for path in TEMP_FILES:
            try:
                os.remove(path)
            except Exception as e:
                print("Unexpected exception:", e)

    sys.exit(exit_code)

def run(cmd, **kwargs):
    print("Running:", cmd)
    return subprocess.run(cmd, **kwargs)

def decode(f):
    return f.decode(encoding="utf-8", errors="ignore")

TEMP_FILES = []

def new_temp_file(prefix, suffix=None):
    temp = NamedTemporaryFile(delete=False, prefix=prefix, suffix=suffix)
    temp.close()
    TEMP_FILES.append(temp.name)
    return temp.name

def contains(pattern, lines):
    for line in lines:
        if re.match(pattern, line):
            return True
    return False

def regexp(pattern):
    if not pattern.startswith("^"):
        pattern = ".*" + pattern
    if not pattern.endswith("$"):
        pattern = pattern + ".*"
    return re.compile(pattern)

def run_bindgen(args, bindings):
    manifest_path = os.path.abspath(os.path.join(os.path.dirname(sys.argv[0]),
                                                 "..",
                                                 "Cargo.toml"))
    command = ["cargo", "run", "--manifest-path", manifest_path]
    if args.release:
        command += ["--release"]
    command += ["--", args.input, "-o", bindings]
    command += shlex.split(args.bindgen_args)

    child = run(command, stderr=subprocess.PIPE, stdout=subprocess.PIPE)

    if args.bindgen_grep:
        pattern = regexp(args.bindgen_grep)
        if not (contains(pattern, decode(child.stdout).splitlines()) or
                contains(pattern, decode(child.stderr).splitlines())):
            exit_1("Error: did not find '{}' in `bindgen`'s output".format(args.bindgen_grep), child)

    if args.expect_bindgen_fail and child.returncode == 0:
        exit_1("Error: expected running `bindgen` to fail, but it didn't", child)

    if not args.expect_bindgen_fail and child.returncode != 0:
        exit_1("Error: running `bindgen` failed", child)

    for arg in args.bindings_grep or []:
        pattern = regexp(arg)
        with open(bindings, mode="r") as f:
            if not contains(pattern, f):
                print("Error: expected the emitted bindings to contain '{}', but they didn't".format(arg))
                print("---------- {} ----------------------------------------------".format(bindings))
                f.seek(0)
                print(f.read())
                raise ExitOne()

def run_rustc(args, bindings, test_exe):
    if args.extra_compile_file:
        with open(bindings, mode="a") as outfile:
            with open(args.extra_compile_file, mode="r") as infile:
                outfile.write(infile.read())
    child = run(
        ["rustc", "--crate-type", "lib", "--test", "-o", test_exe, bindings],
        stdout=subprocess.PIPE,
        stderr=subprocess.PIPE)

    if args.rustc_grep:
        pattern = regexp(args.rustc_grep)
        if not (contains(pattern, decode(child.stdout).splitlines()) or
                contains(pattern, decode(child.stderr).splitlines())):
            exit_1("Error: did not find '{}' in `rustc`'s output".format(args.rustc_grep), child)

    if args.expect_compile_fail and child.returncode == 0:
        exit_1("Error: expected running `rustc` on the emitted bindings to fail, but it didn't", child)

    if not args.expect_compile_fail and child.returncode != 0:
        exit_1("Error: running `rustc` on the emitted bindings failed", child)

def run_layout_tests(args, test_exe):
    child = run(
        [test_exe],
        stdout=subprocess.PIPE,
        stderr=subprocess.PIPE)

    if args.layout_tests_grep:
        pattern = regexp(args.layout_tests_grep)
        if not (contains(pattern, decode(child.stdout).splitlines()) or
                contains(pattern, decode(child.stderr).splitlines())):
            exit_1("Error: did not find '{}' in the compiled bindings' layout tests' output".format(args.layout_tests_grep), child)

    if args.expect_layout_tests_fail and child.returncode == 0:
        exit_1("Error: expected running the compiled bindings' layout tests to fail, but it didn't", child)

    if not args.expect_layout_tests_fail and child.returncode != 0:
        exit_1("Error: running the compiled bindings' layout tests failed", child)

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before>from django.shortcuts import render
from django.template.loader import get_template
from django.http import HttpResponse
from .forms import newMember
from ematrikeln.models import *
from django.shortcuts import redirect

def index(req):
    allMembers = User.objects.all()
    return(render(req,'index.html',{'all':allMembers}))

def add(req,state):
    if state is '':
        success = 'false'
    else:
        success = 'true'
    form = newMember()
    return(render(req,'new.html',{'showdialog':success,'form':form}))

def view_member(req,req_id):
    member =  User.objects.get(id=req_id)
    return(render(req,'medlem.html',{'user':member}))
def delete_member(req,req_id):
    member =  User.objects.get(id= req_id)
    member.delete()
    return redirect('/')
def add_member(req):
    if req.method == 'POST':
        if req.is_valid():
            firstName   = req.POST['firstName']
            lastName    = req.POST['lastName']
            address     = req.POST['address']
            phone       = req.POST['phonenumber']
            city        = req.POST['city']
            postcode    = req.POST['postcode']
            epost       = req.POST['email']
            if req.POST['gymnasium'] != 'Annat':
                gymnasium = req.POST['gymnasium']
            else:
                gymnasium = req.POST['gymnasiumAnnat']
            if req.POST['hometown'] != 'Annat':
                hometown = req.POST['hometown']
            else:
                hometown = req.POST['hometownAnnat']
            if req.POST['school'] != 'Annat':
                school = req.POST['school']
            else:
                school = req.POST['schoolAnnat']
            study = req.POST['study']
            endYear = req.POST['endYear']
            startYear = req.POST['startYear']
            active = req.POST['active']
            # medlemsavgift = req.POST['Betalat']
            studyLine = StudyLine.objects.get_or_create(school = School.objects.get_or_create(name=school)[0], name = study)
            study = Study.objects.create(startYear = startYear,graduated = False,active=active,endYear=endYear, studyLine = studyLine[0])
            newMember = User.objects.create(homeTown = Town.objects.get_or_create(name=hometown)[0], gymnasium = Gymnasium.objects.get_or_create(name=gymnasium)[0], study=study)
            newMember.firstName = firstName
            newMember.lastName = lastName
            newMember.phone = phone
            newMember.adress = address
            newMember.city = city
            newMember.email = epost
            newMember.study = study
            newMember.postcode = postcode
            newMember.save()
            return HttpResponse(200)
        return HttpResponse(500)
    return HttpResponse(404)
<commit_msg>back to modalchoice<commit_after>from django.shortcuts import render
from django.template.loader import get_template
from django.http import HttpResponse
from .forms import newMember
from ematrikeln.models import *
from django.shortcuts import redirect

def index(req):
    allMembers = User.objects.all()
    return(render(req,'index.html',{'all':allMembers}))

def dict2list(list1,list2,list3,other):
    res1 = []
    res2 = []
    res3 = []
    for i in list1:
        print(i)
        res1.append((i['id'],i['name']))
    for i in list2:
        res2.append((i['id'],i['name']))
    for i in list3:
        res3.append((i['id'],i['name']))
    res1.sort()
    res1.append(('-1',other))
    res2.sort()
    res2.append(('-1',other))
    res3.sort()
    res3.append(('-1',other))
    return [res1,res2,res3]

def add(req,state):
    if state is '':
        success = 'false'
    else:
        success = 'true'

    hometownlist    = Town.objects.all()
    schoollist      = School.objects.all()
    gymnasiumlist   = Gymnasium.objects.all()
    herp = dict2list(hometownlist.values(),gymnasiumlist.values(),schoollist.values(),"Annat")
    print(herp)
    form = newMember(*herp)
    return(render(req,'new.html',{'showdialog':success,'form':form}))

def view_member(req,req_id):
    member =  User.objects.get(id=req_id)
    return(render(req,'medlem.html',{'user':member}))
def delete_member(req,req_id):
    member =  User.objects.get(id= req_id)
    member.delete()
    return redirect('/')
def add_member(req):
    if req.method == 'POST':
        if req.is_valid():
            firstName   = req.POST['firstName']
            lastName    = req.POST['lastName']
            address     = req.POST['address']
            phone       = req.POST['phonenumber']
            city        = req.POST['city']
            postcode    = req.POST['postcode']
            epost       = req.POST['email']
            if req.POST['gymnasium'] != 'Annat':
                gymnasium = req.POST['gymnasium']
            else:
                gymnasium = req.POST['gymnasiumAnnat']
            if req.POST['hometown'] != 'Annat':
                hometown = req.POST['hometown']
            else:
                hometown = req.POST['hometownAnnat']
            if req.POST['school'] != 'Annat':
                school = req.POST['school']
            else:
                school = req.POST['schoolAnnat']
            study = req.POST['study']
            endYear = req.POST['endYear']
            startYear = req.POST['startYear']
            active = req.POST['active']
            # medlemsavgift = req.POST['Betalat']
            studyLine = StudyLine.objects.get_or_create(school = School.objects.get_or_create(name=school)[0], name = study)
            study = Study.objects.create(startYear = startYear,graduated = False,active=active,endYear=endYear, studyLine = studyLine[0])
            newMember = User.objects.create(homeTown = Town.objects.get_or_create(name=hometown)[0], gymnasium = Gymnasium.objects.get_or_create(name=gymnasium)[0], study=study)
            newMember.firstName = firstName
            newMember.lastName = lastName
            newMember.phone = phone
            newMember.adress = address
            newMember.city = city
            newMember.email = epost
            newMember.study = study
            newMember.postcode = postcode
            newMember.save()
            return HttpResponse(200)
        return HttpResponse(500)
    return HttpResponse(404)
<|endoftext|>


<commit_before>from osv import fields, osv
from openerp.tools.translate import _

import logging

logger = logging.getLogger(__name__)

class analytic_timesheet_task(osv.osv):
    _name="hr.analytic.timesheet"
    _inherit="hr.analytic.timesheet"

    # Delete method override
    def unlink(self, cr, uid, ids, context=None):
        logger.log(logging.INFO, "deleting task work")
        lineObj = self.pool.get('hr.analytic.timesheet')
        line = lineObj.browse(cr, uid, ids, context)[0]
        if line.task_work_line_id != 0:
            # Delete work from task too
            self.pool.get('project.task.work').unlink(cr, uid, line.task_work_line_id, context)
        res = super(analytic_timesheet_task, self).unlink(cr, uid, ids, context=context)
        return res

    def create(self, cr, uid, vals, context=None):
        logger.log(logging.INFO, "creating task work")
        print str(vals)
        # Create task work
        project = vals.get('project_id') or False
        task = vals.get('task') or False
        #print vals['project_id']
        #print vals['task']
        if project or task:
            logger.log(logging.INFO, "creating new work line for task")
            description =  vals['name']
            date = vals['date']
            unit_amount = vals['unit_amount']
            taskWorkObj = self.pool.get('project.task.work')

            workVals = {
                'name': description,
                'project_id': project,
                'task_id': task,
                'date': date,
                'hours': unit_amount,
                'user_id': uid,
            }

            currentWorkLineID = taskWorkObj.create(cr, uid, workVals, context)
            vals['task_work_line_id'] = currentWorkLineID

        res = super(analytic_timesheet_task, self).create(cr, uid, vals, context=context)
        return res

    def write(self, cr, uid, ids, vals, context=None):
        logger.log(logging.INFO, "updating task work")
        print str(ids)
        print str(vals)

        lineObj = self.pool.get('hr.analytic.timesheet')
        line = lineObj.browse(cr, uid, ids, context)[0]
        currentWorkLineID = line.task_work_line_id
        taskWorkObj = self.pool.get('project.task.work')

        description =  vals.get('name') or line.name
        date = vals.get('date') or line.date
        unit_amount = vals.get('unit_amount') or line.unit_amount
        project = vals.get('project_id') or False
        task = vals.get('name') or False

        workVals = {
            'name': description,
            'project_id': project,
            'task_id': task,
            'date': date,
            'hours': unit_amount,
            'user_id': uid,
        }

        # if project or task has changed remove old work from previous task
        if project or task:
            logger.log(logging.INFO, "project or task changed")
            # remove old task work and create new one
            taskWorkObj.unlink(cr, uid, currentWorkLineID, context)
            currentWorkLineID = taskWorkObj.create(cr, uid, workVals, context)
            vals['task_work_line_id'] = currentWorkLineID
        else:
            # write to old task work
            taskWorkObj.write(cr, uid, currentWorkLineID, workVals, context)

        res = super(analytic_timesheet_task, self).write(cr, uid, ids, vals, context=context)
        return res

    _columns={
        'task_work_line_id': fields.integer('Task work line in the selected task'),
        'project_id': fields.many2one('project.project', 'Project'),
        'task': fields.many2one('project.task', 'Project Task'),
    }

    _defaults = {
        'task_work_line_id': 0,
    }

analytic_timesheet_task()
<commit_msg>debugging work line creation<commit_after>from osv import fields, osv
from openerp.tools.translate import _

import logging

logger = logging.getLogger(__name__)

class analytic_timesheet_task(osv.osv):
    _name="hr.analytic.timesheet"
    _inherit="hr.analytic.timesheet"

    # Delete method override
    def unlink(self, cr, uid, ids, context=None):
        logger.log(logging.INFO, "deleting task work")
        lineObj = self.pool.get('hr.analytic.timesheet')
        line = lineObj.browse(cr, uid, ids, context)[0]
        if line.task_work_line_id != 0:
            # Delete work from task too
            self.pool.get('project.task.work').unlink(cr, uid, line.task_work_line_id, context)
        res = super(analytic_timesheet_task, self).unlink(cr, uid, ids, context=context)
        return res

    def create(self, cr, uid, vals, context=None):
        logger.log(logging.INFO, "creating task work")
        print str(vals)
        # Create task work
        project = vals.get('project_id') or False
        task = vals.get('task') or False
        #print vals['project_id']
        #print vals['task']
        if project or task:
            logger.log(logging.INFO, "creating new work line for task")
            description =  vals['name']
            date = vals['date']
            unit_amount = vals['unit_amount']
            taskWorkObj = self.pool.get('project.task.work')

            workVals = {
                'name': description,
                'project_id': project,
                'task_id': task,
                'date': date,
                'hours': unit_amount,
                'user_id': uid,
            }

            currentWorkLineID = taskWorkObj.create(cr, uid, workVals, context)
            vals['task_work_line_id'] = currentWorkLineID

        res = super(analytic_timesheet_task, self).create(cr, uid, vals, context=context)
        return res

    def write(self, cr, uid, ids, vals, context=None):
        logger.log(logging.INFO, "updating task work")
        print str(ids)
        print str(vals)

        lineObj = self.pool.get('hr.analytic.timesheet')
        line = lineObj.browse(cr, uid, ids, context)[0]
        currentWorkLineID = line.task_work_line_id
        taskWorkObj = self.pool.get('project.task.work')

        description =  vals.get('name') or line.name
        date = vals.get('date') or line.date
        unit_amount = vals.get('unit_amount') or line.unit_amount
        project = vals.get('project_id') or False
        task = vals.get('name') or False

        workVals = {
            'name': description,
            'project_id': project,
            'task_id': task,
            'date': date,
            'hours': unit_amount,
            'user_id': uid,
        }

        # if project or task has changed remove old work from previous task
        if project or task:
            logger.log(logging.INFO, "project or task changed")
            # remove old task work and create new one
            taskWorkObj.unlink(cr, uid, currentWorkLineID, context)
            currentWorkLineID = taskWorkObj.create(cr, uid, workVals, context)
            vals['task_work_line_id'] = currentWorkLineID
        elif currentWorkLineID != 0:
            # write to old task work
            taskWorkObj.write(cr, uid, currentWorkLineID, workVals, context)

        res = super(analytic_timesheet_task, self).write(cr, uid, ids, vals, context=context)
        return res

    _columns={
        'task_work_line_id': fields.integer('Task work line in the selected task'),
        'project_id': fields.many2one('project.project', 'Project'),
        'task': fields.many2one('project.task', 'Project Task'),
    }

    _defaults = {
        'task_work_line_id': 0,
    }

analytic_timesheet_task()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
##############################################################################
#
#    Authors: Nemry Jonathan
#    Copyright (c) 2014 Acsone SA/NV (http://www.acsone.eu)
#    All Rights Reserved
#
#    WARNING: This program as such is intended to be used by professional
#    programmers who take the whole responsibility of assessing all potential
#    consequences resulting from its eventual inadequacies and bugs.
#    End users who are looking for a ready-to-use solution with commercial
#    guarantees and support are strongly advised to contact a Free Software
#    Service Company.
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as
#    published by the Free Software Foundation, either version 3 of the
#    License, or (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################
{
    'name': 'Easy Debug Backend',
    'version': '1.0',
    'author': 'ACSONE SA/NV',
    'maintainer': 'ACSONE SA/NV',
    'website': 'http://www.acsone.eu',
    'category': 'Other',
    'depends': [
        'base',
    ],
    'description': """
Easy Debug Backend
==================
Constantly apply 'debug mode' for backend
""",
    'data': [
        'views/easy_debug_backend.xml',
    ],
    'installable': True,
    'auto_install': False,
}
<commit_msg>[FIX][easy_debug_backend] add missing dependency<commit_after># -*- coding: utf-8 -*-
##############################################################################
#
#    Authors: Nemry Jonathan
#    Copyright (c) 2014 Acsone SA/NV (http://www.acsone.eu)
#    All Rights Reserved
#
#    WARNING: This program as such is intended to be used by professional
#    programmers who take the whole responsibility of assessing all potential
#    consequences resulting from its eventual inadequacies and bugs.
#    End users who are looking for a ready-to-use solution with commercial
#    guarantees and support are strongly advised to contact a Free Software
#    Service Company.
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as
#    published by the Free Software Foundation, either version 3 of the
#    License, or (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################
{
    'name': 'Easy Debug Backend',
    'version': '1.0',
    'author': 'ACSONE SA/NV',
    'maintainer': 'ACSONE SA/NV',
    'website': 'http://www.acsone.eu',
    'category': 'Other',
    'depends': [
        'base',
        'web',
    ],
    'description': """
Easy Debug Backend
==================
Constantly apply 'debug mode' for backend
""",
    'data': [
        'views/easy_debug_backend.xml',
    ],
    'installable': True,
    'auto_install': False,
}
<|endoftext|>


<commit_before>"""
Tests for the analytical.utils module.
"""
# import django

from django.contrib.auth.models import AbstractBaseUser
from django.db import models
from django.http import HttpRequest
from django.template import Context
from django.test.utils import override_settings

from analytical.utils import (
    AnalyticalException,
    get_domain,
    get_identity,
    get_required_setting,
    is_internal_ip,
)
from analytical.tests.utils import TestCase

class SettingDeletedTestCase(TestCase):

    @override_settings(USER_ID=None)
    def test_get_required_setting(self):
        """
        Make sure using get_required_setting fails in the right place.
        """

        # available in python >= 3.2
        if hasattr(self, 'assertRaisesRegex'):
            with self.assertRaisesRegex(AnalyticalException, "^USER_ID setting is not set$"):
                get_required_setting("USER_ID", "\d+", "invalid USER_ID")
        # available in python >= 2.7, deprecated in 3.2
        elif hasattr(self, 'assertRaisesRegexp'):
            with self.assertRaisesRegexp(AnalyticalException, "^USER_ID setting is not set$"):
                get_required_setting("USER_ID", "\d+", "invalid USER_ID")
        else:
            self.assertRaises(AnalyticalException,
                              get_required_setting, "USER_ID", "\d+", "invalid USER_ID")

class MyUser(AbstractBaseUser):
    identity = models.CharField(max_length=50)
    USERNAME_FIELD = 'identity'

class GetIdentityTestCase(TestCase):
    def test_custom_username_field(self):
        get_id = get_identity(Context({}), user=MyUser(identity='fake_id'))
        self.assertEqual(get_id, 'fake_id')

@override_settings(ANALYTICAL_DOMAIN="example.org")
class GetDomainTestCase(TestCase):
    def test_get_service_domain_from_context(self):
        context = Context({'test_domain': 'example.com'})
        self.assertEqual(get_domain(context, 'test'), 'example.com')

    def test_get_analytical_domain_from_context(self):
        context = Context({'analytical_domain': 'example.com'})
        self.assertEqual(get_domain(context, 'test'), 'example.com')

    @override_settings(TEST_DOMAIN="example.net")
    def test_get_service_domain_from_settings(self):
        context = Context()
        self.assertEqual(get_domain(context, 'test'), 'example.net')

    def test_get_analytical_domain_from_settings(self):
        context = Context()
        self.assertEqual(get_domain(context, 'test'), 'example.org')

# FIXME: enable Django apps dynamically and enable test again
# @with_apps('django.contrib.sites')
# @override_settings(TEST_DOMAIN=SETTING_DELETED, ANALYTICAL_DOMAIN=SETTING_DELETED)
# class GetDomainTestCaseWithSites(TestCase):
#    def test_get_domain_from_site(self):
#        site = Site.objects.create(domain="example.com", name="test")
#        with override_settings(SITE_ID=site.id):
#            context = Context()
#            self.assertEqual(get_domain(context, 'test'), 'example.com')

class InternalIpTestCase(TestCase):

    @override_settings(ANALYTICAL_INTERNAL_IPS=['1.1.1.1'])
    def test_render_no_internal_ip(self):
        context = Context()
        self.assertFalse(is_internal_ip(context))

    @override_settings(INTERNAL_IPS=['1.1.1.1'])
    @override_settings(ANALYTICAL_INTERNAL_IPS=[])
    def test_render_analytical_internal_ips_override_when_empty(self):
        req = HttpRequest()
        req.META['REMOTE_ADDR'] = '1.1.1.1'
        context = Context({'request': req})
        self.assertFalse(is_internal_ip(context))

    @override_settings(ANALYTICAL_INTERNAL_IPS=['1.1.1.1'])
    def test_render_internal_ip(self):
        req = HttpRequest()
        req.META['REMOTE_ADDR'] = '1.1.1.1'
        context = Context({'request': req})
        self.assertTrue(is_internal_ip(context))

    @override_settings(TEST_INTERNAL_IPS=['1.1.1.1'])
    def test_render_prefix_internal_ip(self):
        req = HttpRequest()
        req.META['REMOTE_ADDR'] = '1.1.1.1'
        context = Context({'request': req})
        self.assertTrue(is_internal_ip(context, 'TEST'))

    @override_settings(INTERNAL_IPS=['1.1.1.1'])
    def test_render_internal_ip_fallback(self):
        req = HttpRequest()
        req.META['REMOTE_ADDR'] = '1.1.1.1'
        context = Context({'request': req})
        self.assertTrue(is_internal_ip(context))

    @override_settings(ANALYTICAL_INTERNAL_IPS=['1.1.1.1'])
    def test_render_internal_ip_forwarded_for(self):
        req = HttpRequest()
        req.META['HTTP_X_FORWARDED_FOR'] = '1.1.1.1'
        context = Context({'request': req})
        self.assertTrue(is_internal_ip(context))

    @override_settings(ANALYTICAL_INTERNAL_IPS=['1.1.1.1'])
    def test_render_different_internal_ip(self):
        req = HttpRequest()
        req.META['REMOTE_ADDR'] = '2.2.2.2'
        context = Context({'request': req})
        self.assertFalse(is_internal_ip(context))
<commit_msg>Address flake8 W605 (invalid escape sequence)<commit_after>"""
Tests for the analytical.utils module.
"""
# import django

from django.contrib.auth.models import AbstractBaseUser
from django.db import models
from django.http import HttpRequest
from django.template import Context
from django.test.utils import override_settings

from analytical.utils import (
    AnalyticalException,
    get_domain,
    get_identity,
    get_required_setting,
    is_internal_ip,
)
from analytical.tests.utils import TestCase

class SettingDeletedTestCase(TestCase):

    @override_settings(USER_ID=None)
    def test_get_required_setting(self):
        """
        Make sure using get_required_setting fails in the right place.
        """

        # available in python >= 3.2
        if hasattr(self, 'assertRaisesRegex'):
            with self.assertRaisesRegex(AnalyticalException, "^USER_ID setting is not set$"):
                get_required_setting("USER_ID", r"\d+", "invalid USER_ID")
        # available in python >= 2.7, deprecated in 3.2
        elif hasattr(self, 'assertRaisesRegexp'):
            with self.assertRaisesRegexp(AnalyticalException, "^USER_ID setting is not set$"):
                get_required_setting("USER_ID", r"\d+", "invalid USER_ID")
        else:
            self.assertRaises(AnalyticalException,
                              get_required_setting, "USER_ID", r"\d+", "invalid USER_ID")

class MyUser(AbstractBaseUser):
    identity = models.CharField(max_length=50)
    USERNAME_FIELD = 'identity'

class GetIdentityTestCase(TestCase):
    def test_custom_username_field(self):
        get_id = get_identity(Context({}), user=MyUser(identity='fake_id'))
        self.assertEqual(get_id, 'fake_id')

@override_settings(ANALYTICAL_DOMAIN="example.org")
class GetDomainTestCase(TestCase):
    def test_get_service_domain_from_context(self):
        context = Context({'test_domain': 'example.com'})
        self.assertEqual(get_domain(context, 'test'), 'example.com')

    def test_get_analytical_domain_from_context(self):
        context = Context({'analytical_domain': 'example.com'})
        self.assertEqual(get_domain(context, 'test'), 'example.com')

    @override_settings(TEST_DOMAIN="example.net")
    def test_get_service_domain_from_settings(self):
        context = Context()
        self.assertEqual(get_domain(context, 'test'), 'example.net')

    def test_get_analytical_domain_from_settings(self):
        context = Context()
        self.assertEqual(get_domain(context, 'test'), 'example.org')

# FIXME: enable Django apps dynamically and enable test again
# @with_apps('django.contrib.sites')
# @override_settings(TEST_DOMAIN=SETTING_DELETED, ANALYTICAL_DOMAIN=SETTING_DELETED)
# class GetDomainTestCaseWithSites(TestCase):
#    def test_get_domain_from_site(self):
#        site = Site.objects.create(domain="example.com", name="test")
#        with override_settings(SITE_ID=site.id):
#            context = Context()
#            self.assertEqual(get_domain(context, 'test'), 'example.com')

class InternalIpTestCase(TestCase):

    @override_settings(ANALYTICAL_INTERNAL_IPS=['1.1.1.1'])
    def test_render_no_internal_ip(self):
        context = Context()
        self.assertFalse(is_internal_ip(context))

    @override_settings(INTERNAL_IPS=['1.1.1.1'])
    @override_settings(ANALYTICAL_INTERNAL_IPS=[])
    def test_render_analytical_internal_ips_override_when_empty(self):
        req = HttpRequest()
        req.META['REMOTE_ADDR'] = '1.1.1.1'
        context = Context({'request': req})
        self.assertFalse(is_internal_ip(context))

    @override_settings(ANALYTICAL_INTERNAL_IPS=['1.1.1.1'])
    def test_render_internal_ip(self):
        req = HttpRequest()
        req.META['REMOTE_ADDR'] = '1.1.1.1'
        context = Context({'request': req})
        self.assertTrue(is_internal_ip(context))

    @override_settings(TEST_INTERNAL_IPS=['1.1.1.1'])
    def test_render_prefix_internal_ip(self):
        req = HttpRequest()
        req.META['REMOTE_ADDR'] = '1.1.1.1'
        context = Context({'request': req})
        self.assertTrue(is_internal_ip(context, 'TEST'))

    @override_settings(INTERNAL_IPS=['1.1.1.1'])
    def test_render_internal_ip_fallback(self):
        req = HttpRequest()
        req.META['REMOTE_ADDR'] = '1.1.1.1'
        context = Context({'request': req})
        self.assertTrue(is_internal_ip(context))

    @override_settings(ANALYTICAL_INTERNAL_IPS=['1.1.1.1'])
    def test_render_internal_ip_forwarded_for(self):
        req = HttpRequest()
        req.META['HTTP_X_FORWARDED_FOR'] = '1.1.1.1'
        context = Context({'request': req})
        self.assertTrue(is_internal_ip(context))

    @override_settings(ANALYTICAL_INTERNAL_IPS=['1.1.1.1'])
    def test_render_different_internal_ip(self):
        req = HttpRequest()
        req.META['REMOTE_ADDR'] = '2.2.2.2'
        context = Context({'request': req})
        self.assertFalse(is_internal_ip(context))
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

"""
    DU task for BAR documents - see https://read02.uibk.ac.at/wiki/index.php/Document_Understanding_BAR

    Here we convert the human annotation into 2 kinds of annotations:
    - a semantic one: header, heading, page-number, resolution-marginalia, resolution-number, resolution-paragraph   (we ignore Marginalia because only 2 occurences)
    - a segmentation one: 2 complementary labels. We call them Heigh Ho. Could have been Yin Yang as well...
    - also, we store the resolution number in @DU_num

    These new annotations are stored in @DU_sem , @DU_sgm , @DU_num

    Copyright Naver Labs(C) 2017 JL Meunier

    This program is free software: you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
    the Free Software Foundation, either version 3 of the License, or
    (at your option) any later version.

    This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU General Public License for more details.

    You sSegmHould have received a copy of the GNU General Public License
    along with this program.  If not, see <http://www.gnu.org/licenses/>.

    Developed  for the EU project READ. The READ project has received funding
    from the European Union's Horizon 2020 research and innovation programme
    under grant agreement No 674943.

"""
import sys, os, re
import libxml2

try: #to ease the use without proper Python installation
    import TranskribusDU_version
except ImportError:
    sys.path.append( os.path.dirname(os.path.dirname( os.path.abspath(sys.argv[0]) )) )
    import TranskribusDU_version

from common.trace import traceln

from xml_formats.PageXml import PageXml, MultiPageXml, PageXmlException
from crf.Graph_MultiPageXml import Graph_MultiPageXml
from util.Polygon import Polygon

class DU_BAR_Convert:
    """
    Here we convert the human annotation into 2 kinds of annotations:
    - a semantic one: header, heading, page-number, resolution-marginalia, resolution-number, resolution-paragraph   (we ignore Marginalia because only 2 occurences)
    - a segmentation one: 2 complementary labels. We call them Heigh Ho. Could have been Yin Yang as well...

    These new annotations are store in @DU_sem and @DU_sgm
    """
    sXml_HumanAnnotation_Extension   = ".mpxml"
    sXml_MachineAnnotation_Extension = ".du_mpxml"

    sMetadata_Creator  = "TranskribusDU/usecases/BAR/DU_ConvertGTAnnotation.py"
    sMetadata_Comments = "Converted human annotation into semantic and segmentation annotation. See attributes @DU_sem and @DU_sgm."

    dNS = {"pc":PageXml.NS_PAGE_XML}
    sxpNode = ".//pc:TextRegion"

    #Name of attributes for semantic / segmentation /resolution number
    sSemAttr = "DU_sem"
    sSgmAttr = "DU_sgm"
    sNumAttr = "DU_num"

    sOther = "other"

    #Mapping to new semantic annotation
    dAnnotMapping = {"header"       :"header",
                     "heading"      :"heading",
                     "page-number"  :"page-number",
                     "marginalia"   : sOther,
                     "p"            :"resolution-paragraph",
                     "m"            :"resolution-marginalia",
                     ""             :"resolution-number"
                     }
    creResolutionHumanLabel = re.compile("([mp]?)([0-9]+.?)")  #e.g. p1 m23 456 456a

    #The two complementary segmentation labels
    sSegmHeigh = "heigh"
    sSegmHo    = "ho"

    #=== CONFIGURATION ====================================================================
    def __init__(self):

        pass

    def convertDoc(self, sFilename):

        assert sFilename.endswith(self.sXml_HumanAnnotation_Extension)

        g = Graph_MultiPageXml()

        doc = libxml2.parseFile(sFilename)

        #the Heigh/Ho annotation runs over consecutive pages, so we keep those values accross pages
        self.prevResolutionNumber, self.prevSgmLbl = None, None
        self.lSeenResoNum = list()

        for pnum, page, domNdPage in g._iter_Page_DomNode(doc):
            self._convertPageAnnotation(pnum, page, domNdPage)

        MultiPageXml.setMetadata(doc, None, self.sMetadata_Creator, self.sMetadata_Comments)

        assert sFilename.endswith(self.sXml_HumanAnnotation_Extension)

        sDUFilename = sFilename[:-len(self.sXml_HumanAnnotation_Extension)] + self.sXml_MachineAnnotation_Extension
        doc.saveFormatFileEnc(sDUFilename, "utf-8", True)  #True to indent the XML
        doc.freeDoc()

        return sDUFilename

    # -----------------------------------------------------------------------------------------------------------

    def _getNextSegmentationLabel(self, sPrevSegmLabel=None):
        """
        alternate beween HEIGH and HO, 1st at random
        """
        if   sPrevSegmLabel == self.sSegmHeigh: return self.sSegmHo
        elif sPrevSegmLabel == self.sSegmHo:    return self.sSegmHeigh
        else:
            assert sPrevSegmLabel == None
            return self.sSegmHeigh

    def _iter_TextRegionNodeTop2Bottom(self, domNdPage, page):
        """
        Get the DOM, the DOM page node, the page object

        iterator on the DOM, that returns nodes
        """
        #--- XPATH contexts
        ctxt = domNdPage.doc.xpathNewContext()
        for ns, nsurl in self.dNS.items(): ctxt.xpathRegisterNs(ns, nsurl)

        assert self.sxpNode, "CONFIG ERROR: need an xpath expression to enumerate elements corresponding to graph nodes"
        ctxt.setContextNode(domNdPage)
        lNdBlock = ctxt.xpathEval(self.sxpNode) #all relevant nodes of the page

        #order blocks from top to bottom of page
        lOrderedNdBlock = list()
        for ndBlock in lNdBlock:

            lXY = PageXml.getPointList(ndBlock)  #the polygon
            if lXY == []:
                raise ValueError("Node %x has invalid coordinates" % str(ndBlock))

            plg = Polygon(lXY)
            _, (xg, yg) = plg.getArea_and_CenterOfMass()

            lOrderedNdBlock.append( (yg, ndBlock))  #we want to order from top to bottom, so that TextRegions of different resolution are not interleaved

        lOrderedNdBlock.sort()

        for _, ndBlock in lOrderedNdBlock: yield ndBlock

        ctxt.xpathFreeContext()

        raise StopIteration()

    def _convertPageAnnotation(self, pnum, page, domNdPage):
        """

        """
        for nd in self._iter_TextRegionNodeTop2Bottom(domNdPage, page):

            try:
                lbl = PageXml.getCustomAttr(nd, "structure", "type")
            except PageXmlException:
                nd.setProp(self.sSemAttr, self.sOther)
                nd.setProp(self.sSgmAttr, self.sOther)
                continue    #this node has no annotation whatsoever

            if lbl in ["heading", "header", "page-number", "marginalia"]:
                semLabel = lbl
                sgmLabel = self.sOther #those elements are not part of a resolution
                sResoNum = None
            else:
                o = self.creResolutionHumanLabel.match(lbl)
                if not o: raise ValueError("%s is not a valid human annotation" % lbl)
                semLabel = o.group(1)   #"" for the resolution number

                #now decide on the segmentation label
                sResoNum = o.group(2)
                if not sResoNum: raise ValueError("%s is not a valid human annotation - missing resolution number" % lbl)

                #now switch between heigh and ho !! :))
                if self.prevResolutionNumber == sResoNum:
                    sgmLabel = self.prevSgmLbl
                else:
                    sgmLabel = self._getNextSegmentationLabel(self.prevSgmLbl)
                    assert sResoNum not in self.lSeenResoNum, "ERROR: the ordering of the block has not preserved resolution number contiguity"
                    self.lSeenResoNum.append(sResoNum)

                self.prevResolutionNumber, self.prevSgmLbl = sResoNum, sgmLabel

            #always have a semantic label
            sNewSemLbl = self.dAnnotMapping[semLabel]
            assert sNewSemLbl
            nd.setProp(self.sSemAttr, sNewSemLbl)       #DU annotation

            #resolution parts also have a segmentation label and a resolution number
            assert sgmLabel
            nd.setProp(self.sSgmAttr, sgmLabel)     #DU annotation

            if sResoNum:
                nd.setProp(self.sNumAttr, sResoNum)

def test_RE():
    cre = DU_BAR_Convert.creResolutionHumanLabel

    o = cre.match("m103a")
    assert  o.group(1) == 'm'
    assert  o.group(2) == '103a'

    o = cre.match("103a")
    assert  o.group(1) == ''
    assert  o.group(2) == '103a'

    o = cre.match("103")
    assert  o.group(1) == ''
    assert  o.group(2) == '103'

    o = cre.match("az103a")
    assert o == None

if __name__ == "__main__":
    from optparse import OptionParser

    #prepare for the parsing of the command line
    parser = OptionParser(usage="BAR annotation conversion", version="1.0")

#     parser.add_option("--tst", dest='lTst',  action="append", type="string"
#                       , help="Test a model using the given annotated collection.")
#     parser.add_option("--fold-init", dest='iFoldInitNum',  action="store", type="int"
#                       , help="Initialize the file lists for parallel cross-validating a model on the given annotated collection. Indicate the number of folds.")
#     parser.add_option("--jgjhg", dest='bFoldFinish',  action="store_true"
#                       , help="Evaluate by cross-validation a model on the given annotated collection.")
#     parser.add_option("-w", "--warm", dest='warm',  action="store_true"
#                       , help="To make warm-startable model and warm-start if a model exist already.")

    #parse the command line
    (options, args) = parser.parse_args()

    # ---
    doer = DU_BAR_Convert()
    for sFilename in args:
        print "- Processing %s" % sFilename
        sOutputFilename = doer.convertDoc(sFilename)
        print "   done --> %s" %  sOutputFilename

    print "DONE."

<commit_msg>eacvh page start with Heigh<commit_after># -*- coding: utf-8 -*-

"""
    DU task for BAR documents - see https://read02.uibk.ac.at/wiki/index.php/Document_Understanding_BAR

    Here we convert the human annotation into 2 kinds of annotations:
    - a semantic one: header, heading, page-number, resolution-marginalia, resolution-number, resolution-paragraph   (we ignore Marginalia because only 2 occurences)
    - a segmentation one: 2 complementary labels. We call them Heigh Ho. Could have been Yin Yang as well...
    - also, we store the resolution number in @DU_num

    These new annotations are stored in @DU_sem , @DU_sgm , @DU_num

    Copyright Naver Labs(C) 2017 JL Meunier

    This program is free software: you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
    the Free Software Foundation, either version 3 of the License, or
    (at your option) any later version.

    This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU General Public License for more details.

    You sSegmHould have received a copy of the GNU General Public License
    along with this program.  If not, see <http://www.gnu.org/licenses/>.

    Developed  for the EU project READ. The READ project has received funding
    from the European Union's Horizon 2020 research and innovation programme
    under grant agreement No 674943.

"""
import sys, os, re
import libxml2

try: #to ease the use without proper Python installation
    import TranskribusDU_version
except ImportError:
    sys.path.append( os.path.dirname(os.path.dirname( os.path.abspath(sys.argv[0]) )) )
    import TranskribusDU_version

from common.trace import traceln

from xml_formats.PageXml import PageXml, MultiPageXml, PageXmlException
from crf.Graph_MultiPageXml import Graph_MultiPageXml
from util.Polygon import Polygon

class DU_BAR_Convert:
    """
    Here we convert the human annotation into 2 kinds of annotations:
    - a semantic one: header, heading, page-number, resolution-marginalia, resolution-number, resolution-paragraph   (we ignore Marginalia because only 2 occurences)
    - a segmentation one: 2 complementary labels. We call them Heigh Ho. Could have been Yin Yang as well...

    These new annotations are store in @DU_sem and @DU_sgm
    """
    sXml_HumanAnnotation_Extension   = ".mpxml"
    sXml_MachineAnnotation_Extension = ".du_mpxml"

    sMetadata_Creator  = "TranskribusDU/usecases/BAR/DU_ConvertGTAnnotation.py"
    sMetadata_Comments = "Converted human annotation into semantic and segmentation annotation. See attributes @DU_sem and @DU_sgm."

    dNS = {"pc":PageXml.NS_PAGE_XML}
    sxpNode = ".//pc:TextRegion"

    #Name of attributes for semantic / segmentation /resolution number
    sSemAttr = "DU_sem"
    sSgmAttr = "DU_sgm"
    sNumAttr = "DU_num"

    sOther = "other"

    #Mapping to new semantic annotation
    dAnnotMapping = {"header"       :"header",
                     "heading"      :"heading",
                     "page-number"  :"page-number",
                     "marginalia"   : sOther,
                     "p"            :"resolution-paragraph",
                     "m"            :"resolution-marginalia",
                     ""             :"resolution-number"
                     }
    creResolutionHumanLabel = re.compile("([mp]?)([0-9]+.?)")  #e.g. p1 m23 456 456a

    #The two complementary segmentation labels
    sSegmHeigh = "heigh"
    sSegmHo    = "ho"

    #=== CONFIGURATION ====================================================================
    def __init__(self):

        pass

    def convertDoc(self, sFilename):

        assert sFilename.endswith(self.sXml_HumanAnnotation_Extension)

        g = Graph_MultiPageXml()

        doc = libxml2.parseFile(sFilename)

        #the Heigh/Ho annotation runs over consecutive pages, so we keep those values accross pages
        self._initSegmentationLabel()
        self.lSeenResoNum = list()

        for pnum, page, domNdPage in g._iter_Page_DomNode(doc):
            self._convertPageAnnotation(pnum, page, domNdPage)

        MultiPageXml.setMetadata(doc, None, self.sMetadata_Creator, self.sMetadata_Comments)

        assert sFilename.endswith(self.sXml_HumanAnnotation_Extension)

        sDUFilename = sFilename[:-len(self.sXml_HumanAnnotation_Extension)] + self.sXml_MachineAnnotation_Extension
        doc.saveFormatFileEnc(sDUFilename, "utf-8", True)  #True to indent the XML
        doc.freeDoc()

        return sDUFilename

    # -----------------------------------------------------------------------------------------------------------

    def _initSegmentationLabel(self):
        self.prevResolutionNumber, self.prevSgmLbl = None, None

    def _getNextSegmentationLabel(self, sPrevSegmLabel=None):
        """
        alternate beween HEIGH and HO, 1st at random
        """
        if   sPrevSegmLabel == self.sSegmHeigh: return self.sSegmHo
        elif sPrevSegmLabel == self.sSegmHo:    return self.sSegmHeigh
        else:
            assert sPrevSegmLabel == None
            return self.sSegmHeigh

    def _iter_TextRegionNodeTop2Bottom(self, domNdPage, page):
        """
        Get the DOM, the DOM page node, the page object

        iterator on the DOM, that returns nodes
        """
        #--- XPATH contexts
        ctxt = domNdPage.doc.xpathNewContext()
        for ns, nsurl in self.dNS.items(): ctxt.xpathRegisterNs(ns, nsurl)

        assert self.sxpNode, "CONFIG ERROR: need an xpath expression to enumerate elements corresponding to graph nodes"
        ctxt.setContextNode(domNdPage)
        lNdBlock = ctxt.xpathEval(self.sxpNode) #all relevant nodes of the page

        #order blocks from top to bottom of page
        lOrderedNdBlock = list()
        for ndBlock in lNdBlock:

            lXY = PageXml.getPointList(ndBlock)  #the polygon
            if lXY == []:
                raise ValueError("Node %x has invalid coordinates" % str(ndBlock))

            plg = Polygon(lXY)
            _, (xg, yg) = plg.getArea_and_CenterOfMass()

            lOrderedNdBlock.append( (yg, ndBlock))  #we want to order from top to bottom, so that TextRegions of different resolution are not interleaved

        lOrderedNdBlock.sort()

        for _, ndBlock in lOrderedNdBlock: yield ndBlock

        ctxt.xpathFreeContext()

        raise StopIteration()

    def _convertPageAnnotation(self, pnum, page, domNdPage):
        """

        """

        #change: on each page we start by Heigh
        bRestartAtEachPageWithHeigh = True
        if bRestartAtEachPageWithHeigh: self._initSegmentationLabel()

        for nd in self._iter_TextRegionNodeTop2Bottom(domNdPage, page):

            try:
                lbl = PageXml.getCustomAttr(nd, "structure", "type")
            except PageXmlException:
                nd.setProp(self.sSemAttr, self.sOther)
                nd.setProp(self.sSgmAttr, self.sOther)
                continue    #this node has no annotation whatsoever

            if lbl in ["heading", "header", "page-number", "marginalia"]:
                semLabel = lbl
                sgmLabel = self.sOther #those elements are not part of a resolution
                sResoNum = None
            else:
                o = self.creResolutionHumanLabel.match(lbl)
                if not o: raise ValueError("%s is not a valid human annotation" % lbl)
                semLabel = o.group(1)   #"" for the resolution number

                #now decide on the segmentation label
                sResoNum = o.group(2)
                if not sResoNum: raise ValueError("%s is not a valid human annotation - missing resolution number" % lbl)

                #now switch between heigh and ho !! :))
                if self.prevResolutionNumber == sResoNum:
                    sgmLabel = self.prevSgmLbl
                else:
                    sgmLabel = self._getNextSegmentationLabel(self.prevSgmLbl)
                    assert bRestartAtEachPageWithHeigh or sResoNum not in self.lSeenResoNum, "ERROR: the ordering of the block has not preserved resolution number contiguity"
                    self.lSeenResoNum.append(sResoNum)

                self.prevResolutionNumber, self.prevSgmLbl = sResoNum, sgmLabel

            #always have a semantic label
            sNewSemLbl = self.dAnnotMapping[semLabel]
            assert sNewSemLbl
            nd.setProp(self.sSemAttr, sNewSemLbl)       #DU annotation

            #resolution parts also have a segmentation label and a resolution number
            assert sgmLabel
            nd.setProp(self.sSgmAttr, sgmLabel)     #DU annotation

            if sResoNum:
                nd.setProp(self.sNumAttr, sResoNum)

# #not a good idea, I think
# class DU_BAR_Convert_v2(DU_BAR_Convert):
#     """
#     For segmentation labels, we only use 'Heigh' or 'Ho' whatever the semantic label is.
#     (we use to label 'Other' texts that were not part of a resolution)
#     """
#
#     def _initSegmentationLabel(self):
#         self.prevResolutionNumber   = None
#         self.curSgmLbl              = self.sSegmHeigh
#
#     def _switchSegmentationLabel(self):
#         """
#         alternate beween HEIGH and HO, 1st at random
#         """
#         self.curSgmLbl = self.sSegmHeigh if self.curSgmLbl == self.sSegmHo else self.sSegmHo
#
#     def _convertPageAnnotation(self, pnum, page, domNdPage):
#         """
#
#         """
#         for nd in self._iter_TextRegionNodeTop2Bottom(domNdPage, page):
#
#             try:
#                 lbl = PageXml.getCustomAttr(nd, "structure", "type")
#
#                 #
#                 if lbl in ["heading", "header", "page-number", "marginalia"]:
#                     semLabel = self.dAnnotMapping[lbl]
#                     sResoNum = None
#                 else:
#                     o = self.creResolutionHumanLabel.match(lbl)
#                     if not o: raise ValueError("%s is not a valid human annotation" % lbl)
#                     semLabel = self.dAnnotMapping[o.group(1)]   #"" for the resolution number
#
#                     #now decide on the segmentation label
#                     sResoNum = o.group(2)
#                     if not sResoNum: raise ValueError("%s is not a valid human annotation - missing resolution number" % lbl)
#
#                     #now switch between heigh and ho !! :))
#                     if self.prevResolutionNumber != None and self.prevResolutionNumber != sResoNum:
#                         #it is not the first resolution and it has a new number
#                         self._switchSegmentationLabel()
#
#                         assert sResoNum not in self.lSeenResoNum, "ERROR: the ordering of the block has not preserved resolution number contiguity"
#                         self.lSeenResoNum.append(sResoNum)
#
#                     self.prevResolutionNumber = sResoNum
#
#             except PageXmlException:
#                 semLabel = self.sOther
#                 sResoNum = None
#
#             nd.setProp(self.sSemAttr, semLabel)
#             nd.setProp(self.sSgmAttr, self.curSgmLbl)
#             if sResoNum:
#                 nd.setProp(self.sNumAttr, sResoNum)

def test_RE():
    cre = DU_BAR_Convert.creResolutionHumanLabel

    o = cre.match("m103a")
    assert  o.group(1) == 'm'
    assert  o.group(2) == '103a'

    o = cre.match("103a")
    assert  o.group(1) == ''
    assert  o.group(2) == '103a'

    o = cre.match("103")
    assert  o.group(1) == ''
    assert  o.group(2) == '103'

    o = cre.match("az103a")
    assert o == None

if __name__ == "__main__":
    from optparse import OptionParser

    #prepare for the parsing of the command line
    parser = OptionParser(usage="BAR annotation conversion", version="1.0")

#     parser.add_option("--tst", dest='lTst',  action="append", type="string"
#                       , help="Test a model using the given annotated collection.")
#     parser.add_option("--fold-init", dest='iFoldInitNum',  action="store", type="int"
#                       , help="Initialize the file lists for parallel cross-validating a model on the given annotated collection. Indicate the number of folds.")
#     parser.add_option("--jgjhg", dest='bFoldFinish',  action="store_true"
#                       , help="Evaluate by cross-validation a model on the given annotated collection.")
#     parser.add_option("-w", "--warm", dest='warm',  action="store_true"
#                       , help="To make warm-startable model and warm-start if a model exist already.")

    #parse the command line
    (options, args) = parser.parse_args()

    # ---
    doer = DU_BAR_Convert()
    for sFilename in args:
        print "- Processing %s" % sFilename
        sOutputFilename = doer.convertDoc(sFilename)
        print "   done --> %s" %  sOutputFilename

    print "DONE."

<|endoftext|>