<commit_before># Copyright (c) 2012 Lars Hupfeldt Nielsen, Hupfeldt IT
# All rights reserved. This work is under a BSD license, see LICENSE.TXT.

from pytest import xfail

from multiconf import mc_config
from multiconf.envs import EnvFactory

from .utils.tstclasses import ItemWithAA

ef_dev_prod = EnvFactory()

dev2cta = ef_dev_prod.Env('dev2cta')
dev2sta = ef_dev_prod.Env('dev2sta')
dev2tta = ef_dev_prod.Env('dev2tta')

dev2ctb = ef_dev_prod.Env('dev2ctb')
dev2stb = ef_dev_prod.Env('dev2stb')
dev2ttb = ef_dev_prod.Env('dev2ttb')

g_dev2ctst1 = ef_dev_prod.EnvGroup('g_dev2ctst1', dev2cta, dev2sta, dev2ctb)  # No dev2stb
g_dev2ctst2 = ef_dev_prod.EnvGroup('g_dev2ctst2', dev2sta, dev2ctb, dev2stb)  # No dev2cta
g_dev2 = ef_dev_prod.EnvGroup('g_dev2', g_dev2ctst1, dev2tta, dev2ttb)

dev3ct = ef_dev_prod.Env('dev3ct')
dev3st = ef_dev_prod.Env('dev3st')
dev3tt = ef_dev_prod.Env('dev3tt')
g_dev3ctst = ef_dev_prod.EnvGroup('g_dev3ctst', dev3ct, dev3st)
g_dev3 = ef_dev_prod.EnvGroup('g_dev3', g_dev3ctst, dev3tt)

g_dev_overlap1 = ef_dev_prod.EnvGroup('g_dev_overlap1', dev2cta)
g_dev_overlap2 = ef_dev_prod.EnvGroup('g_dev_overlap2', dev2cta)

g_dev_overlap23ct = ef_dev_prod.EnvGroup('g_dev_overlap23ct', dev2cta, dev3ct)
g_dev_overlap23st = ef_dev_prod.EnvGroup('g_dev_overlap23st', dev2sta, dev3st)
g_dev_overlap23ctst = ef_dev_prod.EnvGroup('g_dev_overlap23ctst', dev2cta, dev3st)

pp = ef_dev_prod.Env('pp')
prod = ef_dev_prod.Env('prod')
g_prod = ef_dev_prod.EnvGroup('g_prod', pp, prod)

def test_value_defined_through_three_groups_resolved_immediately():
    @mc_config(ef_dev_prod)
    def config(root):
        with ItemWithAA() as cr:
            cr.setattr('aa', g_dev_overlap2=7, default=7, dev2cta=15, prod=1, g_dev2=2, g_dev_overlap1=3)

    cr = ef_dev_prod.config(prod).ItemWithAA
    assert cr.aa == 1
    cr = ef_dev_prod.config(dev2cta).ItemWithAA
    assert cr.aa == 15

def test_value_defined_through_three_groups_resolved_immediately_multiple1():
    @mc_config(ef_dev_prod)
    def config(root):
        with ItemWithAA() as cr:
            cr.setattr('aa', g_dev_overlap23ct=7, default=7, dev2cta=15, prod=1, g_dev2=2, g_dev_overlap23st=3, dev2sta=8)

    cr = ef_dev_prod.config(prod).ItemWithAA
    assert cr.aa == 1
    cr = ef_dev_prod.config(dev2cta).ItemWithAA
    assert cr.aa == 15

def test_value_defined_through_three_groups_resolved_immediately_multiple2():
    @mc_config(ef_dev_prod)
    def config(root):
        with ItemWithAA() as cr:
            cr.setattr('aa', g_dev_overlap23ct=7, default=7, dev2cta=15, prod=1, g_dev2=2, g_dev_overlap23st=3, dev2sta=8, g_dev_overlap23ctst=23, dev3st=0)
            with ItemWithAA() as it1:
                it1.setattr('aa', g_dev_overlap23ct=7, default=7, prod=1, g_dev_overlap23st=3, g_dev_overlap23ctst=23, g_dev2=2, dev3st=0, dev2sta=8, dev2cta=15)
                with ItemWithAA() as it2:
                    it2.setattr('aa', g_dev_overlap23ct=7, g_dev_overlap23st=3, g_dev_overlap23ctst=23, g_dev2=2, dev3st=0, dev2sta=8, dev2cta=15, default=7, prod=1)
                    with ItemWithAA() as it3:
                        it3.setattr('aa', default=7, prod=1, g_dev_overlap23st=3, g_dev_overlap23ctst=23, g_dev2=2, dev3st=0, dev2sta=8, dev2cta=15)

    cr = ef_dev_prod.config(prod).ItemWithAA
    assert cr.aa == 1
    assert cr.ItemWithAA.aa == 1
    assert cr.ItemWithAA.ItemWithAA.aa == 1
    assert cr.ItemWithAA.ItemWithAA.ItemWithAA.aa == 1

    cr = ef_dev_prod.config(dev2cta).ItemWithAA
    assert cr.aa == 15
    assert cr.ItemWithAA.aa == 15
    assert cr.ItemWithAA.ItemWithAA.aa == 15
    assert cr.ItemWithAA.ItemWithAA.ItemWithAA.aa == 15

# Yes, **kwargs are not ordered, so these are just named differently, can give another order
# just to make sure we don't depend on the order
ef2_x_prod = EnvFactory()

x2ct2 = ef2_x_prod.Env('x2ct')
x2st2 = ef2_x_prod.Env('x2st')
x2tt2 = ef2_x_prod.Env('x2tt')
g_x2ctst2 = ef2_x_prod.EnvGroup('g_x2ctst', x2ct2, x2st2)
g_x22 = ef2_x_prod.EnvGroup('g_x2', g_x2ctst2, x2tt2)

x3ct2 = ef2_x_prod.Env('x3ct')
x3st2 = ef2_x_prod.Env('x3st')
x3tt2 = ef2_x_prod.Env('x3tt')
g_x3ctst2 = ef2_x_prod.EnvGroup('g_x3ctst', x3ct2, x3st2)
g_x32 = ef2_x_prod.EnvGroup('g_x3', g_x3ctst2, x3tt2)

g_x_overlap12 = ef2_x_prod.EnvGroup('g_x_overlap12', x2ct2)
g_x_overlap22 = ef2_x_prod.EnvGroup('g_x_overlap22', x2ct2)

g_x_overlap23ct2 = ef2_x_prod.EnvGroup('g_x_overlap23ct', x2ct2, x3ct2)
g_x_overlap23st2 = ef2_x_prod.EnvGroup('g_x_overlap23st', x2st2, x3st2)
g_x_overlap23ctst2 = ef2_x_prod.EnvGroup('g_x_overlap23ctst', x2ct2, x3st2)

pp2 = ef2_x_prod.Env('pp')
prod2 = ef2_x_prod.Env('prod')
g_prod2 = ef2_x_prod.EnvGroup('g_prod', pp2, prod2)

def test_value_defined_through_three_groups_resolved_multiple_x():
    @mc_config(ef2_x_prod)
    def config(root):
        with ItemWithAA() as cr:
            cr.setattr('aa', g_x_overlap23ct=7, default=7, x2st=8, prod=1, g_x2ctst=2, g_x_overlap23st=3, g_x_overlap23ctst=23, x3st=0, x2ct=15)
            with ItemWithAA() as it:
                it.setattr('aa', g_x_overlap23ct=7, default=7, prod=1, g_x_overlap23st=3, g_x_overlap23ctst=23, g_x2ctst=2, x3st=0, x2ct=15, x2st=8)
                with ItemWithAA() as it:
                    it.setattr('aa', g_x_overlap23ct=7, g_x_overlap23st=3, g_x_overlap23ctst=23, g_x2ctst=2, x3st=0, x2ct=15, x2st=8, default=7, prod=1)
                    with ItemWithAA() as it:
                        it.setattr('aa', default=7, prod=1, g_x_overlap23st=3, g_x_overlap23ctst=23, g_x2ctst=2, x3st=0, x2ct=15, x2st=8, g_x_overlap23ct=7)

    ef2_x_prod.config(prod2)
    ef2_x_prod.config(x2ct2)
    xfail('TODO')
<commit_msg>Minor cleanup<commit_after># Copyright (c) 2012 Lars Hupfeldt Nielsen, Hupfeldt IT
# All rights reserved. This work is under a BSD license, see LICENSE.TXT.

from multiconf import mc_config
from multiconf.envs import EnvFactory

from .utils.tstclasses import ItemWithAA

ef_dev_prod = EnvFactory()

dev2cta = ef_dev_prod.Env('dev2cta')
dev2sta = ef_dev_prod.Env('dev2sta')
dev2tta = ef_dev_prod.Env('dev2tta')

dev2ctb = ef_dev_prod.Env('dev2ctb')
dev2stb = ef_dev_prod.Env('dev2stb')
dev2ttb = ef_dev_prod.Env('dev2ttb')

g_dev2ctst1 = ef_dev_prod.EnvGroup('g_dev2ctst1', dev2cta, dev2sta, dev2ctb)  # No dev2stb
g_dev2ctst2 = ef_dev_prod.EnvGroup('g_dev2ctst2', dev2sta, dev2ctb, dev2stb)  # No dev2cta
g_dev2 = ef_dev_prod.EnvGroup('g_dev2', g_dev2ctst1, dev2tta, dev2ttb)

dev3ct = ef_dev_prod.Env('dev3ct')
dev3st = ef_dev_prod.Env('dev3st')
dev3tt = ef_dev_prod.Env('dev3tt')
g_dev3ctst = ef_dev_prod.EnvGroup('g_dev3ctst', dev3ct, dev3st)
g_dev3 = ef_dev_prod.EnvGroup('g_dev3', g_dev3ctst, dev3tt)

g_dev_overlap1 = ef_dev_prod.EnvGroup('g_dev_overlap1', dev2cta)
g_dev_overlap2 = ef_dev_prod.EnvGroup('g_dev_overlap2', dev2cta)

g_dev_overlap23ct = ef_dev_prod.EnvGroup('g_dev_overlap23ct', dev2cta, dev3ct)
g_dev_overlap23st = ef_dev_prod.EnvGroup('g_dev_overlap23st', dev2sta, dev3st)
g_dev_overlap23ctst = ef_dev_prod.EnvGroup('g_dev_overlap23ctst', dev2cta, dev3st)

pp = ef_dev_prod.Env('pp')
prod = ef_dev_prod.Env('prod')
g_prod = ef_dev_prod.EnvGroup('g_prod', pp, prod)

def test_value_defined_through_three_groups_resolved_immediately():
    @mc_config(ef_dev_prod)
    def config(root):
        with ItemWithAA() as cr:
            cr.setattr('aa', g_dev_overlap2=7, default=7, dev2cta=15, prod=1, g_dev2=2, g_dev_overlap1=3)

    cr = ef_dev_prod.config(prod).ItemWithAA
    assert cr.aa == 1
    cr = ef_dev_prod.config(dev2cta).ItemWithAA
    assert cr.aa == 15

def test_value_defined_through_three_groups_resolved_immediately_multiple1():
    @mc_config(ef_dev_prod)
    def config(root):
        with ItemWithAA() as cr:
            cr.setattr('aa', g_dev_overlap23ct=7, default=7, dev2cta=15, prod=1, g_dev2=2, g_dev_overlap23st=3, dev2sta=8)

    cr = ef_dev_prod.config(prod).ItemWithAA
    assert cr.aa == 1
    cr = ef_dev_prod.config(dev2cta).ItemWithAA
    assert cr.aa == 15

def test_value_defined_through_three_groups_resolved_immediately_multiple2():
    @mc_config(ef_dev_prod)
    def config(root):
        with ItemWithAA() as cr:
            cr.setattr('aa', g_dev_overlap23ct=7, default=7, dev2cta=15, prod=1, g_dev2=2, g_dev_overlap23st=3, dev2sta=8, g_dev_overlap23ctst=23, dev3st=0)
            with ItemWithAA() as it1:
                it1.setattr('aa', g_dev_overlap23ct=7, default=7, prod=1, g_dev_overlap23st=3, g_dev_overlap23ctst=23, g_dev2=2, dev3st=0, dev2sta=8, dev2cta=15)
                with ItemWithAA() as it2:
                    it2.setattr('aa', g_dev_overlap23ct=7, g_dev_overlap23st=3, g_dev_overlap23ctst=23, g_dev2=2, dev3st=0, dev2sta=8, dev2cta=15, default=7, prod=1)
                    with ItemWithAA() as it3:
                        it3.setattr('aa', default=7, prod=1, g_dev_overlap23st=3, g_dev_overlap23ctst=23, g_dev2=2, dev3st=0, dev2sta=8, dev2cta=15)

    cr = ef_dev_prod.config(prod).ItemWithAA
    assert cr.aa == 1
    assert cr.ItemWithAA.aa == 1
    assert cr.ItemWithAA.ItemWithAA.aa == 1
    assert cr.ItemWithAA.ItemWithAA.ItemWithAA.aa == 1

    cr = ef_dev_prod.config(dev2cta).ItemWithAA
    assert cr.aa == 15
    assert cr.ItemWithAA.aa == 15
    assert cr.ItemWithAA.ItemWithAA.aa == 15
    assert cr.ItemWithAA.ItemWithAA.ItemWithAA.aa == 15

# Yes, **kwargs are not ordered, so these are just named differently, can give another order
# just to make sure we don't depend on the order
ef2_x_prod = EnvFactory()

x2ct2 = ef2_x_prod.Env('x2ct')
x2st2 = ef2_x_prod.Env('x2st')
x2tt2 = ef2_x_prod.Env('x2tt')
g_x2ctst2 = ef2_x_prod.EnvGroup('g_x2ctst', x2ct2, x2st2)
g_x22 = ef2_x_prod.EnvGroup('g_x2', g_x2ctst2, x2tt2)

x3ct2 = ef2_x_prod.Env('x3ct')
x3st2 = ef2_x_prod.Env('x3st')
x3tt2 = ef2_x_prod.Env('x3tt')
g_x3ctst2 = ef2_x_prod.EnvGroup('g_x3ctst', x3ct2, x3st2)
g_x32 = ef2_x_prod.EnvGroup('g_x3', g_x3ctst2, x3tt2)

g_x_overlap12 = ef2_x_prod.EnvGroup('g_x_overlap12', x2ct2)
g_x_overlap22 = ef2_x_prod.EnvGroup('g_x_overlap22', x2ct2)

g_x_overlap23ct2 = ef2_x_prod.EnvGroup('g_x_overlap23ct', x2ct2, x3ct2)
g_x_overlap23st2 = ef2_x_prod.EnvGroup('g_x_overlap23st', x2st2, x3st2)
g_x_overlap23ctst2 = ef2_x_prod.EnvGroup('g_x_overlap23ctst', x2ct2, x3st2)

pp2 = ef2_x_prod.Env('pp')
prod2 = ef2_x_prod.Env('prod')
g_prod2 = ef2_x_prod.EnvGroup('g_prod', pp2, prod2)

def test_value_defined_through_three_groups_resolved_multiple_x():
    @mc_config(ef2_x_prod)
    def config(root):
        with ItemWithAA() as cr:
            cr.setattr('aa', g_x_overlap23ct=7, default=7, x2st=8, prod=1, g_x2ctst=2, g_x_overlap23st=3, g_x_overlap23ctst=23, x3st=0, x2ct=15)
            with ItemWithAA() as it:
                it.setattr('aa', g_x_overlap23ct=7, default=7, prod=1, g_x_overlap23st=3, g_x_overlap23ctst=23, g_x2ctst=2, x3st=0, x2ct=15, x2st=8)
                with ItemWithAA() as it:
                    it.setattr('aa', g_x_overlap23ct=7, g_x_overlap23st=3, g_x_overlap23ctst=23, g_x2ctst=2, x3st=0, x2ct=15, x2st=8, default=7, prod=1)
                    with ItemWithAA() as it:
                        it.setattr('aa', default=7, prod=1, g_x_overlap23st=3, g_x_overlap23ctst=23, g_x2ctst=2, x3st=0, x2ct=15, x2st=8, g_x_overlap23ct=7)

    ef2_x_prod.config(prod2)
    ef2_x_prod.config(x2ct2)
    # TODO test values
<|endoftext|>


<commit_before>__author__ = 'noe'

from featurizer import Featurizer
from feature_reader import FeatureReader
from pca import PCA
from uniform_time_clustering import UniformTimeClustering

import psutil
import numpy as np

class Discretizer(object):

    def __init__(self, trajfiles, topfile):
        """

        :return:
        """
        # create featurizer
        featurizer = Featurizer(topfile)
        # this should be an input!
        sel = featurizer.selCa()
        pairs = featurizer.pairs(sel)
        featurizer.distances(pairs)
        # feature reader
        reader = FeatureReader(trajfiles, topfile, featurizer)
        self.transformers.append(reader)
        # pca output dimension and transformation type should be an input!
        pca = PCA(reader,2)
        self.transformers.append(pca)
        # number of states and clustering type should be an input
        utc = UniformTimeClustering(pca,100)
        self.transformers.append(utc)

        # determine memory requirements
        M = psutil.virtual_memory()[1] # available RAM
        print "available RAM: ",M
        const_mem = long(0)
        mem_per_frame = long(0)
        for trans in self.transformers:
            mem_per_frame += trans.get_memory_per_frame()
            const_mem += trans.get_constant_memory()
        print "per-frame memory requirements: ",mem_per_frame
        chunksize = (M-const_mem) / mem_per_frame # maximum allowed chunk size
        # is this chunksize sufficient to store full trajectories?
        chunksize = min(chunksize, np.max(reader.trajectory_lengths()))
        print "resulting chunk size: ",chunksize
        # set chunksize
        for trans in self.transformers:
            trans.set_chunksize(chunksize)

        # any memory unused? if yes, we can store results
        Mfree = M - const_mem - chunksize * mem_per_frame
        print "free memory: ",Mfree

        # starting from the back of the pipeline, store outputs if possible
        for trans in reversed(self.transformers):
            mem_req_trans = trans.n_frames_total() * trans.get_memory_per_frame()
            if Mfree > mem_req_trans:
                Mfree -= mem_req_trans
                trans.operate_in_memory()
                print "spending ", mem_req_trans, " bytes to operate in main memory: ",trans.describe()

    def run(self):
        # parametrize from the beginning to the end
        for trans in self.transformers:
            trans.parametrize()

<commit_msg>use tica instead of pca<commit_after>from coordinates.coordinate_transformation.tica import TICA
__author__ = 'noe'

from featurizer import Featurizer
from feature_reader import FeatureReader
from pca import PCA
from uniform_time_clustering import UniformTimeClustering

import psutil
import numpy as np

class Discretizer(object):

    def __init__(self, trajfiles, topfile):
        """

        :return:
        """
        # create featurizer
        featurizer = Featurizer(topfile)
        # TODO: move this to a testing function
        # TODO: where to build chain? User has to provide list of T objects?
        #sel = featurizer.selCa()
        sel= [0, 320, 1500]
        pairs = featurizer.pairs(sel)
        featurizer.distances(pairs)
        # feature reader
        reader = FeatureReader(trajfiles, topfile, featurizer)

        self.transformers = []

        self.transformers.append(reader)
        # pca output dimension and transformation type should be an input!
        pca = TICA(reader, lag=10, output_dimension=2)#PCA(reader, 2)
        self.transformers.append(pca)
        # number of states and clustering type should be an input
        utc = UniformTimeClustering(pca, 100)
        self.transformers.append(utc)

        # ----------------

        # determine memory requirements
        M = psutil.virtual_memory()[1]  # available RAM
        print "available RAM: ", M
        const_mem = long(0)
        mem_per_frame = long(0)
        for trans in self.transformers:
            mem_per_frame += trans.get_memory_per_frame()
            const_mem += trans.get_constant_memory()
        print "per-frame memory requirements: ", mem_per_frame
        # maximum allowed chunk size
        print "const mem: ", const_mem
        chunksize = (M - const_mem) / mem_per_frame
        if chunksize < 0:
            raise MemoryError('Not enough memory for desired transformation chain!')
        # is this chunksize sufficient to store full trajectories?
        chunksize = min(chunksize, np.max(reader.trajectory_lengths()))
        print "resulting chunk size: ", chunksize
        # set chunksize
        for trans in self.transformers:
            trans.set_chunksize(chunksize)

        # any memory unused? if yes, we can store results
        Mfree = M - const_mem - chunksize * mem_per_frame
        print "free memory: ", Mfree

        # starting from the back of the pipeline, store outputs if possible
        for trans in reversed(self.transformers):
            mem_req_trans = trans.n_frames_total() * trans.get_memory_per_frame()
            if Mfree > mem_req_trans:
                Mfree -= mem_req_trans
                trans.operate_in_memory()
                print "spending ", mem_req_trans, " bytes to operate in main memory: ",trans.describe()

    def run(self):
        # parametrize from the beginning to the end
        for trans in self.transformers:
            trans.parametrize()

<|endoftext|>


<commit_before>from django.forms import ModelForm, Textarea
from django import forms
from bets.models import MatchBet,TournamentBet, BetPoint, Profile
from tournaments.models import Team
from django.contrib.auth.models import User
from tournaments.models import Match
from django.utils import timezone
import pytz
from .fields import TeamChoiceField

class MatchBetForm( ModelForm ):
    """
    MatchBet form to place a bet on a match
    """
    #matchs              = Match.objects.filter(date__gte=timezone.now())
    def __init__( self, *args, **kwargs ):
        mid = kwargs.pop( 'match', None)
        tid = kwargs.pop( 'tournament', None)
        # check if match is available for betting
        # if match is not None, limit choice to that match
        # else limit choice to only available match (match.date > timezone.now()
        choice = [(None, '-- choose a match --'), ]
        # this is in case we present a form where user can bet on every available match
        choice = forms.ModelChoiceField(queryset=Match.objects.filter( date__gte = timezone.now() ) )
        # if mid defined, this is because user used /mbet_add/tid/match/mid
        if mid is not None:
            choice = forms.ModelChoiceField(queryset=Match.objects.filter( id = mid ) )
        # if tid defined, this is beacause user used /mbet_add/tid
        if tid is not None:
            choice  = forms.ModelChoiceField( queryset = Match.objects.filter( tournament = tid ).filter( date__gte = timezone.now() ) )
        super( MatchBetForm, self ).__init__( *args, **kwargs )
        self.fields['match'] = choice

    class Meta:
        model   = MatchBet
        exclude = ['player', 'points_won']

class TeamListField(forms.ChoiceField):
    teams = [(None, '-- choose a team --'), ]
    #teams   = [(None, '-- choose a team --'), ]
    def __init__(self, *args, **kwargs):
        tid = kwargs.pop('tournament', None)
        super(TeamListField, self).__init__(*args, **kwargs)
        if tid is not None:
            matchs  = Match.objects.filter( tournament = tid )
            for m in matchs:
                t = (m.home_team.name, m.home_team.name)
                if t not in self.teams:
                    self.teams.append(t)
                t = (m.away_team.name, m.away_team.name )
                if t not in self.teams:
                   self.teams.append(t)
        self.choices = self.teams
        #self.widget = forms.Select()
        #self.help_text = ""

    def to_python(self, value):
        return Team.objects.get(name = value)

class TournamentBetForm( ModelForm ):
    """
    TournamentBet form which will be closed at the begining of the first match
    """
    #TODO get every team name of the tournament
    def __init__( self, *args, **kwargs):
        tid     = kwargs.pop( 'tournament', None)
        teams   = [(None, '-- choose a team --'), ]
        if tid is not None:
            matchs  = Match.objects.filter( tournament = tid )
            for m in matchs:
                t = (m.home_team.name, m.home_team.name)
                if t not in teams:
                    teams.append(t)
                t = (m.away_team.name, m.away_team.name )
                if t not in teams:
                   teams.append(t)

        super(TournamentBetForm, self).__init__(*args, **kwargs)
        self.fields['first_team'] =  forms.ModelChoiceField(
            queryset = Team.objects.all()
        )
        self.fields['second_team'] =  forms.ModelChoiceField(
            queryset = Team.objects.all()
        )
        self.fields['third_team'] =  forms.ModelChoiceField(
            queryset = Team.objects.all()
        )
        self.fields['fourth_team'] =  forms.ModelChoiceField(
            queryset = Team.objects.all()
        )
        self.fields['fifth_team'] =  forms.ModelChoiceField(
            queryset = Team.objects.all()
        )
        self.fields['sixth_team'] =  forms.ModelChoiceField(
            queryset = Team.objects.all()
        )

    class Meta:
        model   = TournamentBet
        exclude = ['player', 'tournament', 'points_won', 'created_date', 'modified_date']

class BetPointForm( ModelForm ):
    class Meta:
        model   = BetPoint
        fields  = '__all__'

class UserForm(ModelForm):
    class Meta:
        model   = User
        fields = ('first_name', 'last_name', 'email')

class ProfileForm( ModelForm ):

    tz = forms.ChoiceField(
        label= 'Fuseau Horaire',
        choices=[(t, t) for t in pytz.common_timezones]
        )

    class Meta:
        model   = Profile
        fields  = ['tz']
<commit_msg>[fix] fix error in loading module<commit_after>from django.forms import ModelForm, Textarea
from django import forms
from bets.models import MatchBet,TournamentBet, BetPoint, Profile
from tournaments.models import Team
from django.contrib.auth.models import User
from tournaments.models import Match
from django.utils import timezone
import pytz

class MatchBetForm( ModelForm ):
    """
    MatchBet form to place a bet on a match
    """
    #matchs              = Match.objects.filter(date__gte=timezone.now())
    def __init__( self, *args, **kwargs ):
        mid = kwargs.pop( 'match', None)
        tid = kwargs.pop( 'tournament', None)
        # check if match is available for betting
        # if match is not None, limit choice to that match
        # else limit choice to only available match (match.date > timezone.now()
        choice = [(None, '-- choose a match --'), ]
        # this is in case we present a form where user can bet on every available match
        choice = forms.ModelChoiceField(queryset=Match.objects.filter( date__gte = timezone.now() ) )
        # if mid defined, this is because user used /mbet_add/tid/match/mid
        if mid is not None:
            choice = forms.ModelChoiceField(queryset=Match.objects.filter( id = mid ) )
        # if tid defined, this is beacause user used /mbet_add/tid
        if tid is not None:
            choice  = forms.ModelChoiceField( queryset = Match.objects.filter( tournament = tid ).filter( date__gte = timezone.now() ) )
        super( MatchBetForm, self ).__init__( *args, **kwargs )
        self.fields['match'] = choice

    class Meta:
        model   = MatchBet
        exclude = ['player', 'points_won']

class TeamListField(forms.ChoiceField):
    teams = [(None, '-- choose a team --'), ]
    #teams   = [(None, '-- choose a team --'), ]
    def __init__(self, *args, **kwargs):
        tid = kwargs.pop('tournament', None)
        super(TeamListField, self).__init__(*args, **kwargs)
        if tid is not None:
            matchs  = Match.objects.filter( tournament = tid )
            for m in matchs:
                t = (m.home_team.name, m.home_team.name)
                if t not in self.teams:
                    self.teams.append(t)
                t = (m.away_team.name, m.away_team.name )
                if t not in self.teams:
                   self.teams.append(t)
        self.choices = self.teams
        #self.widget = forms.Select()
        #self.help_text = ""

    def to_python(self, value):
        return Team.objects.get(name = value)

class TournamentBetForm( ModelForm ):
    """
    TournamentBet form which will be closed at the begining of the first match
    """
    #TODO get every team name of the tournament
    def __init__( self, *args, **kwargs):
        tid     = kwargs.pop( 'tournament', None)
        teams   = [(None, '-- choose a team --'), ]
        if tid is not None:
            matchs  = Match.objects.filter( tournament = tid )
            for m in matchs:
                t = (m.home_team.name, m.home_team.name)
                if t not in teams:
                    teams.append(t)
                t = (m.away_team.name, m.away_team.name )
                if t not in teams:
                   teams.append(t)

        super(TournamentBetForm, self).__init__(*args, **kwargs)
        self.fields['first_team'] =  forms.ModelChoiceField(
            queryset = Team.objects.all()
        )
        self.fields['second_team'] =  forms.ModelChoiceField(
            queryset = Team.objects.all()
        )
        self.fields['third_team'] =  forms.ModelChoiceField(
            queryset = Team.objects.all()
        )
        self.fields['fourth_team'] =  forms.ModelChoiceField(
            queryset = Team.objects.all()
        )
        self.fields['fifth_team'] =  forms.ModelChoiceField(
            queryset = Team.objects.all()
        )
        self.fields['sixth_team'] =  forms.ModelChoiceField(
            queryset = Team.objects.all()
        )

    class Meta:
        model   = TournamentBet
        exclude = ['player', 'tournament', 'points_won', 'created_date', 'modified_date']

class BetPointForm( ModelForm ):
    class Meta:
        model   = BetPoint
        fields  = '__all__'

class UserForm(ModelForm):
    class Meta:
        model   = User
        fields = ('first_name', 'last_name', 'email')

class ProfileForm( ModelForm ):

    tz = forms.ChoiceField(
        label= 'Fuseau Horaire',
        choices=[(t, t) for t in pytz.common_timezones]
        )

    class Meta:
        model   = Profile
        fields  = ['tz']
<|endoftext|>


<commit_before>#!/usr/bin/python
# -*- coding: utf-8 -*-
# libavg - Media Playback Engine.
# Copyright (C) 2003-2008 Ulrich von Zadow
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
#
# Current versions can be found at www.libavg.de
#

import sys
import os
import time

import libavg
from libavg import avg, Point2D
import testcase

g_Player = avg.Player.get()
g_helper = g_Player.getTestHelper()

TEST_RESOLUTION = (160, 120)

class TestAppBase(libavg.AVGApp):
    def requestStop(self, timeout=0):
        g_Player.setTimeout(timeout, g_Player.stop)

    def singleKeyPress(self, char):
        g_helper.fakeKeyEvent(avg.KEYDOWN, ord(char), ord(char), char, ord(char),
                avg.KEYMOD_NONE)
        g_helper.fakeKeyEvent(avg.KEYUP, ord(char), ord(char), char, ord(char),
                avg.KEYMOD_NONE)

class AVGAppTestCase(testcase.AVGTestCase):
    def testMinimal(self):
        class MinimalApp(TestAppBase):
            testInstance = self
            def init(self):
                self.testInstance.assert_(not g_Player.isFullscreen())
                self.requestStop()

        if 'AVG_DEPLOY' in os.environ:
            del os.environ['AVG_DEPLOY']
        MinimalApp.start(resolution=TEST_RESOLUTION)

    def testAvgDeploy(self):
        class FullscreenApp(TestAppBase):
            testInstance = self
            def init(self):
                self.testInstance.assert_(g_Player.isFullscreen())
                rootNodeSize = g_Player.getRootNode().size
                self.testInstance.assert_(rootNodeSize == TEST_RESOLUTION)
                self.requestStop()

        os.environ['AVG_DEPLOY'] = '1'
        FullscreenApp.start(resolution=TEST_RESOLUTION)
        del os.environ['AVG_DEPLOY']

    def testDebugWindowSize(self):
        class DebugwindowApp(TestAppBase):
            testInstance = self
            def init(self):
                self.testInstance.assert_(not g_Player.isFullscreen())
                rootNodeSize = g_Player.getRootNode().size
                self.testInstance.assert_(rootNodeSize == TEST_RESOLUTION)

                # windowSize = g_Player.getWindowResolution()
                # self.testInstance.assert_(windowSize == Point2D(TEST_RESOLUTION) / 2)
                self.requestStop()

        DebugwindowApp.start(resolution=TEST_RESOLUTION,
                debugWindowSize=Point2D(TEST_RESOLUTION) / 2)

    def testScreenshot(self):
        expectedFiles = ['screenshot-000.png', 'screenshot-001.png']

        def cleanup():
            for screenshotFile in expectedFiles[::-1]:
                if os.path.exists(screenshotFile):
                    os.unlink(screenshotFile)

        def checkCallback():
            for screenshotFile in expectedFiles[::-1]:
                if os.path.exists(screenshotFile):
                    avg.Bitmap(screenshotFile)
                else:
                    raise RuntimeError('Cannot find the expected '
                            'screenshot file %s' % screenshotFile)

            g_Player.stop()

        class ScreenshotApp(TestAppBase):
            def init(self):
                self.singleKeyPress('s')
                self.singleKeyPress('s')
                self.timeStarted = time.time()
                self.timerId = g_Player.setOnFrameHandler(self.onFrame)

            def onFrame(self):
                if (os.path.exists(expectedFiles[-1]) or
                        time.time() - self.timeStarted > 1):
                    g_Player.clearInterval(self.timerId)
                    checkCallback()

        cleanup()
        ScreenshotApp.start(resolution=TEST_RESOLUTION)
        cleanup()

    def testGraphs(self):
        class GraphsApp(TestAppBase):
            def init(self):
                self.enableGraphs()

            def enableGraphs(self):
                self.singleKeyPress('f')
                self.singleKeyPress('m')
                g_Player.setTimeout(500, self.disableGraphs)

            def disableGraphs(self):
                self.singleKeyPress('m')
                self.singleKeyPress('f')
                self.requestStop()

        GraphsApp.start(resolution=TEST_RESOLUTION)

    def testToggleKeys(self):
        TOGGLE_KEYS = ['?', 't', 'e']
        class ToggleKeysApp(TestAppBase):
            def init(self):
                self.keys = TOGGLE_KEYS[:]
                g_Player.setTimeout(0, self.nextKey)

            def nextKey(self):
                if not self.keys:
                    g_Player.stop()
                else:
                    key = self.keys.pop()
                    self.singleKeyPress(key)
                    g_Player.setTimeout(0, self.nextKey)

        ToggleKeysApp.start(resolution=TEST_RESOLUTION)

    def testFakeFullscreen(self):
        class FakeFullscreenApp(TestAppBase):
            fakeFullscreen = True
            def init(self):
                g_Player.setTimeout(0, g_Player.stop)

        if os.name == 'nt':
            FakeFullscreenApp.start(resolution=TEST_RESOLUTION)
        else:
            self.assertException(
                    lambda: FakeFullscreenApp.start(resolution=TEST_RESOLUTION))

def avgAppTestSuite(tests):
    availableTests = (
            'testMinimal',
            'testAvgDeploy',
            'testDebugWindowSize',
            'testScreenshot',
            'testGraphs',
            'testToggleKeys',
            'testFakeFullscreen',
    )
    return testcase.createAVGTestSuite(availableTests, AVGAppTestCase, tests)
<commit_msg>Fixed distcheck failure<commit_after>#!/usr/bin/python
# -*- coding: utf-8 -*-
# libavg - Media Playback Engine.
# Copyright (C) 2003-2008 Ulrich von Zadow
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
#
# Current versions can be found at www.libavg.de
#

import sys
import os
import time

import libavg
from libavg import avg, Point2D
import testcase

g_Player = avg.Player.get()
g_helper = g_Player.getTestHelper()

TEST_RESOLUTION = (160, 120)

class TestAppBase(libavg.AVGApp):
    def requestStop(self, timeout=0):
        g_Player.setTimeout(timeout, g_Player.stop)

    def singleKeyPress(self, char):
        g_helper.fakeKeyEvent(avg.KEYDOWN, ord(char), ord(char), char, ord(char),
                avg.KEYMOD_NONE)
        g_helper.fakeKeyEvent(avg.KEYUP, ord(char), ord(char), char, ord(char),
                avg.KEYMOD_NONE)

class AVGAppTestCase(testcase.AVGTestCase):
    def testMinimal(self):
        class MinimalApp(TestAppBase):
            testInstance = self
            def init(self):
                self.testInstance.assert_(not g_Player.isFullscreen())
                self.requestStop()

        if 'AVG_DEPLOY' in os.environ:
            del os.environ['AVG_DEPLOY']
        MinimalApp.start(resolution=TEST_RESOLUTION)

    def testAvgDeploy(self):
        class FullscreenApp(TestAppBase):
            testInstance = self
            def init(self):
                self.testInstance.assert_(g_Player.isFullscreen())
                rootNodeSize = g_Player.getRootNode().size
                self.testInstance.assert_(rootNodeSize == TEST_RESOLUTION)
                self.requestStop()

        os.environ['AVG_DEPLOY'] = '1'
        FullscreenApp.start(resolution=TEST_RESOLUTION)
        del os.environ['AVG_DEPLOY']

    def testDebugWindowSize(self):
        class DebugwindowApp(TestAppBase):
            testInstance = self
            def init(self):
                self.testInstance.assert_(not g_Player.isFullscreen())
                rootNodeSize = g_Player.getRootNode().size
                self.testInstance.assert_(rootNodeSize == TEST_RESOLUTION)

                # windowSize = g_Player.getWindowResolution()
                # self.testInstance.assert_(windowSize == Point2D(TEST_RESOLUTION) / 2)
                self.requestStop()

        DebugwindowApp.start(resolution=TEST_RESOLUTION,
                debugWindowSize=Point2D(TEST_RESOLUTION) / 2)

    def testScreenshot(self):
        testFile = 'tmp.%f' % time.time()
        try:
            open(testFile, 'w')
        except IOError:
            sys.stderr.write('[Skipped] ')
            return
        else:
            os.unlink(testFile)

        expectedFiles = ['screenshot-000.png', 'screenshot-001.png']

        def cleanup():
            for screenshotFile in expectedFiles[::-1]:
                if os.path.exists(screenshotFile):
                    os.unlink(screenshotFile)

        def checkCallback():
            for screenshotFile in expectedFiles[::-1]:
                if os.path.exists(screenshotFile):
                    avg.Bitmap(screenshotFile)
                else:
                    raise RuntimeError('Cannot find the expected '
                            'screenshot file %s' % screenshotFile)

            g_Player.stop()

        class ScreenshotApp(TestAppBase):
            def init(self):
                self.singleKeyPress('s')
                self.singleKeyPress('s')
                self.timeStarted = time.time()
                self.timerId = g_Player.setOnFrameHandler(self.onFrame)

            def onFrame(self):
                if (os.path.exists(expectedFiles[-1]) or
                        time.time() - self.timeStarted > 1):
                    g_Player.clearInterval(self.timerId)
                    checkCallback()

        cleanup()
        ScreenshotApp.start(resolution=TEST_RESOLUTION)
        cleanup()

    def testGraphs(self):
        class GraphsApp(TestAppBase):
            def init(self):
                self.enableGraphs()

            def enableGraphs(self):
                self.singleKeyPress('f')
                self.singleKeyPress('m')
                g_Player.setTimeout(500, self.disableGraphs)

            def disableGraphs(self):
                self.singleKeyPress('m')
                self.singleKeyPress('f')
                self.requestStop()

        GraphsApp.start(resolution=TEST_RESOLUTION)

    def testToggleKeys(self):
        TOGGLE_KEYS = ['?', 't', 'e']
        class ToggleKeysApp(TestAppBase):
            def init(self):
                self.keys = TOGGLE_KEYS[:]
                g_Player.setTimeout(0, self.nextKey)

            def nextKey(self):
                if not self.keys:
                    g_Player.stop()
                else:
                    key = self.keys.pop()
                    self.singleKeyPress(key)
                    g_Player.setTimeout(0, self.nextKey)

        ToggleKeysApp.start(resolution=TEST_RESOLUTION)

    def testFakeFullscreen(self):
        class FakeFullscreenApp(TestAppBase):
            fakeFullscreen = True
            def init(self):
                g_Player.setTimeout(0, g_Player.stop)

        if os.name == 'nt':
            FakeFullscreenApp.start(resolution=TEST_RESOLUTION)
        else:
            self.assertException(
                    lambda: FakeFullscreenApp.start(resolution=TEST_RESOLUTION))

def avgAppTestSuite(tests):
    availableTests = (
            'testMinimal',
            'testAvgDeploy',
            'testDebugWindowSize',
            'testScreenshot',
            'testGraphs',
            'testToggleKeys',
            'testFakeFullscreen',
    )
    return testcase.createAVGTestSuite(availableTests, AVGAppTestCase, tests)
<|endoftext|>


<commit_before>#!/usr/bin/env python3
# -*-coding:UTF-8 -*
"""
    Decoder module

    Dectect Binary and decode it
"""
import time
import os
import redis
import base64
from hashlib import sha1
import magic
import json
import datetime

from pubsublogger import publisher

from Helper import Process
from packages import Item

from lib import Decoded

import re
import signal

class TimeoutException(Exception):
    pass

def timeout_handler(signum, frame):
    raise TimeoutException

signal.signal(signal.SIGALRM, timeout_handler)

def hex_decoder(hexStr):
    #hexStr = ''.join( hex_string.split(" ") )
    return bytes(bytearray([int(hexStr[i:i+2], 16) for i in range(0, len(hexStr), 2)]))

def binary_decoder(binary_string):
    return bytes(bytearray([int(binary_string[i:i+8], 2) for i in range(0, len(binary_string), 8)]))

def base64_decoder(base64_string):
    return base64.b64decode(base64_string)

def decode_string(content, item_id, item_date, encoded_list, decoder_name, encoded_min_size):
    find = False
    for encoded in encoded_list:
        if len(encoded) >=  encoded_min_size:
            decoded_file = decoder_function[decoder_name](encoded)
            find = True

            sha1_string = sha1(decoded_file).hexdigest()
            mimetype = Decoded.get_file_mimetype(decoded_file)
            if not mimetype:
                print(item_id)
                print(sha1_string)
                raise Exception('Invalid mimetype')
            Decoded.save_decoded_file_content(sha1_string, decoded_file, item_date, mimetype=mimetype)
            Decoded.save_item_relationship(sha1_string, item_id)
            Decoded.create_decoder_matadata(sha1_string, item_id, decoder_name)

            #remove encoded from item content
            content = content.replace(encoded, '', 1)

            print('{} : {} - {}'.format(item_id, decoder_name, mimetype))
    if(find):
        set_out_item(decoder_name, item_id)

    return content

def set_out_item(decoder_name, item_id):
    publisher.warning(decoder_name+' decoded')
    #Send to duplicate
    p.populate_set_out(item_id, 'Duplicate')

    msg = 'infoleak:automatic-detection="'+decoder_name+'";{}'.format(item_id)
    p.populate_set_out(msg, 'Tags')

if __name__ == '__main__':
    # If you wish to use an other port of channel, do not forget to run a subscriber accordingly (see launch_logs.sh)
    # Port of the redis instance used by pubsublogger
    publisher.port = 6380
    # Script is the default channel used for the modules.
    publisher.channel = 'Script'

    # Section name in bin/packages/modules.cfg
    config_section = 'Decoder'

    # Setup the I/O queues
    p = Process(config_section)

    serv_metadata = redis.StrictRedis(
        host=p.config.get("ARDB_Metadata", "host"),
        port=p.config.getint("ARDB_Metadata", "port"),
        db=p.config.getint("ARDB_Metadata", "db"),
        decode_responses=True)

    # Sent to the logging a description of the module
    publisher.info("Decoder started")

    regex_binary = '[0-1]{40,}'
    #regex_hex = '(0[xX])?[A-Fa-f0-9]{40,}'
    regex_hex = '[A-Fa-f0-9]{40,}'
    regex_base64 = '(?:[A-Za-z0-9+/]{4}){2,}(?:[A-Za-z0-9+/]{2}[AEIMQUYcgkosw048]=|[A-Za-z0-9+/][AQgw]==)'

    re.compile(regex_binary)
    re.compile(regex_hex)
    re.compile(regex_base64)

    # map decoder function
    decoder_function = {'binary':binary_decoder,'hexadecimal':hex_decoder, 'base64':base64_decoder}

    hex_max_execution_time = p.config.getint("Hex", "max_execution_time")
    binary_max_execution_time = p.config.getint("Binary", "max_execution_time")
    base64_max_execution_time = p.config.getint("Base64", "max_execution_time")

    # list all decoder yith regex,
    decoder_binary = {'name': 'binary', 'regex': regex_binary, 'encoded_min_size': 300, 'max_execution_time': binary_max_execution_time}
    decoder_hexadecimal = {'name': 'hexadecimal', 'regex': regex_hex, 'encoded_min_size': 300, 'max_execution_time': hex_max_execution_time}
    decoder_base64 = {'name': 'base64', 'regex': regex_base64, 'encoded_min_size': 40, 'max_execution_time': base64_max_execution_time}

    decoder_order = [ decoder_base64, decoder_binary, decoder_hexadecimal, decoder_base64]

    for decoder in decoder_order:
        serv_metadata.sadd('all_decoder', decoder['name'])

    # Endless loop getting messages from the input queue
    while True:
        # Get one message from the input queue
        message = p.get_from_set()
        if message is None:

            publisher.debug("{} queue is empty, waiting".format(config_section))
            time.sleep(1)
            continue

        obj_id = Item.get_item_id(message)

        # Do something with the message from the queue
        content = Item.get_item_content(obj_id)
        date = Item.get_item_date(obj_id)

        for decoder in decoder_order: # add threshold and size limit

            # max execution time on regex
            signal.alarm(decoder['max_execution_time'])
            try:
                encoded_list = re.findall(decoder['regex'], content)
            except TimeoutException:
                encoded_list = []
                p.incr_module_timeout_statistic() # add encoder type
                print ("{0} processing timeout".format(obj_id))
                continue
            else:
                signal.alarm(0)

                if(len(encoded_list) > 0):
                    content = decode_string(content, message, date, encoded_list, decoder['name'], decoder['encoded_min_size'])
<commit_msg>perf: use defined compiled regex<commit_after>#!/usr/bin/env python3
# -*-coding:UTF-8 -*
"""
    Decoder module

    Dectect Binary and decode it
"""

##################################
# Import External packages
##################################
import time
import os
import redis
import base64
from hashlib import sha1
import magic
import json
import datetime
from pubsublogger import publisher
import re
import signal
from lib import Decoded

##################################
# Import Project packages
##################################
from module.abstract_module import AbstractModule
from Helper import Process
from packages import Item

class TimeoutException(Exception):
    pass

def timeout_handler(signum, frame):
    raise TimeoutException

signal.signal(signal.SIGALRM, timeout_handler)

class Decoder(AbstractModule):
    """
    Decoder module for AIL framework
    """

    # TODO to lambda expr
    def hex_decoder(self, hexStr):
        #hexStr = ''.join( hex_string.split(" ") )
        return bytes(bytearray([int(hexStr[i:i+2], 16) for i in range(0, len(hexStr), 2)]))

    # TODO to lambda expr
    def binary_decoder(self, binary_string):
        return bytes(bytearray([int(binary_string[i:i+8], 2) for i in range(0, len(binary_string), 8)]))

    # TODO to lambda expr
    def base64_decoder(self, base64_string):
        return base64.b64decode(base64_string)

    def __init__(self):
        super(Decoder, self).__init__(logger_channel="script:decoder")

        serv_metadata = redis.StrictRedis(
            host=self.process.config.get("ARDB_Metadata", "host"),
            port=self.process.config.getint("ARDB_Metadata", "port"),
            db=self.process.config.getint("ARDB_Metadata", "db"),
            decode_responses=True)

        regex_binary = '[0-1]{40,}'
        #regex_hex = '(0[xX])?[A-Fa-f0-9]{40,}'
        regex_hex = '[A-Fa-f0-9]{40,}'
        regex_base64 = '(?:[A-Za-z0-9+/]{4}){2,}(?:[A-Za-z0-9+/]{2}[AEIMQUYcgkosw048]=|[A-Za-z0-9+/][AQgw]==)'

        cmp_regex_binary = re.compile(regex_binary)
        cmp_regex_hex = re.compile(regex_hex)
        cmp_regex_base64 = re.compile(regex_base64)

        # map decoder function
        self.decoder_function = {'binary':self.binary_decoder,'hexadecimal':self.hex_decoder, 'base64':self.base64_decoder}

        hex_max_execution_time = self.process.config.getint("Hex", "max_execution_time")
        binary_max_execution_time = self.process.config.getint("Binary", "max_execution_time")
        base64_max_execution_time = self.process.config.getint("Base64", "max_execution_time")

        # list all decoder with regex,
        decoder_binary = {'name': 'binary', 'regex': cmp_regex_binary, 'encoded_min_size': 300, 'max_execution_time': binary_max_execution_time}
        decoder_hexadecimal = {'name': 'hexadecimal', 'regex': cmp_regex_hex, 'encoded_min_size': 300, 'max_execution_time': hex_max_execution_time}
        decoder_base64 = {'name': 'base64', 'regex': cmp_regex_base64, 'encoded_min_size': 40, 'max_execution_time': base64_max_execution_time}

        self.decoder_order = [ decoder_base64, decoder_binary, decoder_hexadecimal, decoder_base64]

        for decoder in self.decoder_order:
            serv_metadata.sadd('all_decoder', decoder['name'])

        # Waiting time in secondes between to message proccessed
        self.pending_seconds = 1

        # Send module state to logs
        self.redis_logger.info(f'Module {self.module_name} initialized')

    def compute(self, message):

        obj_id = Item.get_item_id(message)

        # Extract info from message
        content = Item.get_item_content(obj_id)
        date = Item.get_item_date(obj_id)

        for decoder in self.decoder_order: # add threshold and size limit
            # max execution time on regex
            signal.alarm(decoder['max_execution_time'])

            try:
                encoded_list = decoder['regex'].findall(content)
            except TimeoutException:
                encoded_list = []
                self.process.incr_module_timeout_statistic() # add encoder type
                self.redis_logger.debug(f"{obj_id} processing timeout")
                continue
            else:
                signal.alarm(0)

                if(len(encoded_list) > 0):
                    content = self.decode_string(content, message, date, encoded_list, decoder['name'], decoder['encoded_min_size'])

    def decode_string(self, content, item_id, item_date, encoded_list, decoder_name, encoded_min_size):
        find = False
        for encoded in encoded_list:
            if len(encoded) >=  encoded_min_size:
                decoded_file = self.decoder_function[decoder_name](encoded)
                find = True

                sha1_string = sha1(decoded_file).hexdigest()
                mimetype = Decoded.get_file_mimetype(decoded_file)
                if not mimetype:
                    self.redis_logger.debug(item_id)
                    self.redis_logger.debug(sha1_string)
                    raise Exception('Invalid mimetype')
                Decoded.save_decoded_file_content(sha1_string, decoded_file, item_date, mimetype=mimetype)
                Decoded.save_item_relationship(sha1_string, item_id)
                Decoded.create_decoder_matadata(sha1_string, item_id, decoder_name)

                #remove encoded from item content
                content = content.replace(encoded, '', 1)

                self.redis_logger.debug(f'{item_id} : {decoder_name} - {mimetype}')
        if(find):
            self.set_out_item(decoder_name, item_id)

        return content

    def set_out_item(self, decoder_name, item_id):

        self.redis_logger.warning(f'{decoder_name} decoded')

        # Send to duplicate
        self.process.populate_set_out(item_id, 'Duplicate')

        # Send to Tags
        msg = f'infoleak:automatic-detection="{decoder_name}";{item_id}'
        self.process.populate_set_out(msg, 'Tags')

if __name__ == '__main__':

    module = Decoder()
    module.run()
<|endoftext|>


<commit_before>__author__ = 'saeedamen' # Saeed Amen / saeed@thalesians.com

#
# Copyright 2015 Thalesians Ltd. - http//www.thalesians.com / @thalesians
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the
# License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#
# See the License for the specific language governing permissions and limitations under the License.
#
"""
LoaderQuandl

Class for reading in data from Quandl into PyThalesians library

"""

# support Quandl 3.x.x
try:
    import quandl as Quandl
except:
    # if import fails use Quandl 2.x.x
    import Quandl

from pythalesians.util.loggermanager import LoggerManager
from pythalesians.market.loaders.lowlevel.loadertemplate import LoaderTemplate
from pythalesians.util.constants import Constants

class LoaderQuandl(LoaderTemplate):

    def __init__(self):
        super(LoaderQuandl, self).__init__()
        self.logger = LoggerManager().getLogger(__name__)

    # implement method in abstract superclass
    def load_ticker(self, time_series_request):
        time_series_request_vendor = self.construct_vendor_time_series_request(time_series_request)

        self.logger.info("Request Quandl data")

        data_frame = self.download_daily(time_series_request_vendor)

        if data_frame is None or data_frame.index is []: return None

        # convert from vendor to Thalesians tickers/fields
        if data_frame is not None:
            returned_tickers = data_frame.columns

        if data_frame is not None:
            # tidy up tickers into a format that is more easily translatable
            returned_tickers = [x.replace(' - Value', '') for x in returned_tickers]
            returned_tickers = [x.replace(' - VALUE', '') for x in returned_tickers]
            returned_tickers = [x.replace('.', '/') for x in returned_tickers]

            fields = self.translate_from_vendor_field(['close' for x in returned_tickers], time_series_request)
            tickers = self.translate_from_vendor_ticker(returned_tickers, time_series_request)

            ticker_combined = []

            for i in range(0, len(fields)):
                ticker_combined.append(tickers[i] + "." + fields[i])

            data_frame.columns = ticker_combined
            data_frame.index.name = 'Date'

        self.logger.info("Completed request from Quandl.")

        return data_frame

    def download_daily(self, time_series_request):
        trials = 0

        data_frame = None

        while(trials < 5):
            try:
                data_frame = Quandl.get(time_series_request.tickers, authtoken=Constants().quandl_api_key, trim_start=time_series_request.start_date,
                            trim_end=time_series_request.finish_date)

                break
            except:
                trials = trials + 1
                self.logger.info("Attempting... " + str(trials) + " request to download from Quandl")

        if trials == 5:
            self.logger.error("Couldn't download from Quandl after several attempts!")

        return data_frame

<commit_msg>Fixed issue with multiple tickers returned by Quandl<commit_after>__author__ = 'saeedamen' # Saeed Amen / saeed@thalesians.com

#
# Copyright 2015 Thalesians Ltd. - http//www.thalesians.com / @thalesians
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the
# License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#
# See the License for the specific language governing permissions and limitations under the License.
#
"""
LoaderQuandl

Class for reading in data from Quandl into PyThalesians library

"""

# support Quandl 3.x.x
try:
    import quandl as Quandl
except:
    # if import fails use Quandl 2.x.x
    import Quandl

from pythalesians.util.loggermanager import LoggerManager
from pythalesians.market.loaders.lowlevel.loadertemplate import LoaderTemplate
from pythalesians.util.constants import Constants

class LoaderQuandl(LoaderTemplate):

    def __init__(self):
        super(LoaderQuandl, self).__init__()
        self.logger = LoggerManager().getLogger(__name__)

    # implement method in abstract superclass
    def load_ticker(self, time_series_request):
        time_series_request_vendor = self.construct_vendor_time_series_request(time_series_request)

        self.logger.info("Request Quandl data")

        data_frame = self.download_daily(time_series_request_vendor)

        if data_frame is None or data_frame.index is []: return None

        # convert from vendor to Thalesians tickers/fields
        if data_frame is not None:
            returned_tickers = data_frame.columns

        if data_frame is not None:
            # tidy up tickers into a format that is more easily translatable
            # we can often get multiple fields returned (even if we don't ask for them!)
            # convert to lower case
            returned_fields = [(x.split(' - ')[1]).lower().replace(' ', '-') for x in returned_tickers]

            returned_tickers = [x.replace('.', '/') for x in returned_tickers]
            returned_tickers = [x.split(' - ')[0] for x in returned_tickers]

            fields = self.translate_from_vendor_field(returned_fields, time_series_request)
            tickers = self.translate_from_vendor_ticker(returned_tickers, time_series_request)

            ticker_combined = []

            for i in range(0, len(fields)):
                ticker_combined.append(tickers[i] + "." + fields[i])

            data_frame.columns = ticker_combined
            data_frame.index.name = 'Date'

        self.logger.info("Completed request from Quandl.")

        return data_frame

    def download_daily(self, time_series_request):
        trials = 0

        data_frame = None

        while(trials < 5):
            try:
                data_frame = Quandl.get(time_series_request.tickers, authtoken=Constants().quandl_api_key, trim_start=time_series_request.start_date,
                            trim_end=time_series_request.finish_date)

                break
            except:
                trials = trials + 1
                self.logger.info("Attempting... " + str(trials) + " request to download from Quandl")

        if trials == 5:
            self.logger.error("Couldn't download from Quandl after several attempts!")

        return data_frame

<|endoftext|>


<commit_before>#-*- coding: utf-8 -*-
"""
This package is an implementation of the OpenID specification in
Python.  It contains code for both server and consumer
implementations.  For information on implementing an OpenID consumer,
see the C{L{openid.consumer.consumer}} module.  For information on
implementing an OpenID server, see the C{L{openid.server.server}}
module.

@contact: U{http://github.com/necaris/python3-openid/}

@copyright: (C) 2005-2008 JanRain, Inc., 2012-2013 Rami Chowdhury

@license: Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.
    You may obtain a copy of the License at
    U{http://www.apache.org/licenses/LICENSE-2.0}

    Unless required by applicable law or agreed to in writing, software
    distributed under the License is distributed on an "AS IS" BASIS,
    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    See the License for the specific language governing permissions
    and limitations under the License.
"""

version_info = (3, 0, 10)

__version__ = ".".join(str(x) for x in version_info)

__all__ = [
    'association',
    'consumer',
    'cryptutil',
    'dh',
    'extension',
    'extensions',
    'fetchers',
    'kvform',
    'message',
    'oidutil',
    'server',
    'sreg',
    'store',
    'urinorm',
    'yadis',
]
<commit_msg>Update copyright info<commit_after>#-*- coding: utf-8 -*-
"""
This package is an implementation of the OpenID specification in
Python.  It contains code for both server and consumer
implementations.  For information on implementing an OpenID consumer,
see the C{L{openid.consumer.consumer}} module.  For information on
implementing an OpenID server, see the C{L{openid.server.server}}
module.

@contact: U{http://github.com/necaris/python3-openid/}

@copyright: (C) 2005-2008 JanRain, Inc., 2012-2017 Rami Chowdhury

@license: Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.
    You may obtain a copy of the License at
    U{http://www.apache.org/licenses/LICENSE-2.0}

    Unless required by applicable law or agreed to in writing, software
    distributed under the License is distributed on an "AS IS" BASIS,
    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    See the License for the specific language governing permissions
    and limitations under the License.
"""

version_info = (3, 0, 10)

__version__ = ".".join(str(x) for x in version_info)

__all__ = [
    'association',
    'consumer',
    'cryptutil',
    'dh',
    'extension',
    'extensions',
    'fetchers',
    'kvform',
    'message',
    'oidutil',
    'server',
    'sreg',
    'store',
    'urinorm',
    'yadis',
]
<|endoftext|>


<commit_before>class Range(object):
    def __init__(self, rid, start, end):
        if end < start:
            raise ValueError("illegal range. end is smaller than start")
        if start == end:
            raise ValueError("range can't be with size 0")
        self.rid, self.start, self.end = rid, start, end

    def __str__(self):
        return "[#%s:%d-%d]" % (self.rid, self.start, self.end)

    def __repr__(self):
        return self.__str__()

    def __eq__(self, other):
        return (other is not None and
                    (self.rid == other.rid and
                     self.start == other.start and
                     self.end == other.end))

    def __ne__(self, other):
        return not self == other

    def contain(self, other):
        return other.start >= self.start and other.end <= self.end

<commit_msg>removing "#" from Range __str__ so it won't look like a comment..<commit_after>class Range(object):
    def __init__(self, rid, start, end):
        if end < start:
            raise ValueError("illegal range. end is smaller than start")
        if start == end:
            raise ValueError("range can't be with size 0")
        self.rid, self.start, self.end = rid, start, end

    def __str__(self):
        return "[%s:%d-%d]" % (self.rid, self.start, self.end)

    def __repr__(self):
        return self.__str__()

    def __eq__(self, other):
        return (other is not None and
                    (self.rid == other.rid and
                     self.start == other.start and
                     self.end == other.end))

    def __ne__(self, other):
        return not self == other

    def contain(self, other):
        return other.start >= self.start and other.end <= self.end

<|endoftext|>


<commit_before># Copyright Hybrid Logic Ltd.  See LICENSE file for details.

"""
Persistence of cluster configuration.
"""

from pyrsistent import PRecord, PVector, PMap, PSet
from json import dumps, loads, JSONEncoder

from twisted.python.filepath import FilePath
from twisted.application.service import Service
from twisted.internet.defer import succeed

from ._model import SERIALIZABLE_CLASSES, Deployment

# Serialization marker storing the class name:
_CLASS_MARKER = u"$__class__$"

class _ConfigurationEncoder(JSONEncoder):
    """
    JSON encoder that can encode the configuration model.
    """
    def default(self, obj):
        if isinstance(obj, PRecord):
            result = dict(obj)
            result[_CLASS_MARKER] = obj.__class__.__name__
            return result
        elif isinstance(obj, PMap):
            return dict(obj)
        elif isinstance(obj, (PSet, PVector, set)):
            return list(obj)
        elif isinstance(obj, FilePath):
            return {_CLASS_MARKER: u"FilePath",
                    u"path": obj.path.decode("utf-8")}
        return JSONEncoder.default(self, obj)

def wire_encode(obj):
    """
    Encode the given configuration object into bytes.

    :param obj: An object from the configuration model, e.g. ``Deployment``.
    :return bytes: Encoded object.
    """
    return dumps(obj, cls=_ConfigurationEncoder)

def wire_decode(data):
    """
    Decode the given configuration object from bytes.

    :param bytes data: Encoded object.
    :param obj: An object from the configuration model, e.g. ``Deployment``.
    """
    classes = {cls.__name__: cls for cls in SERIALIZABLE_CLASSES}

    def decode_object(dictionary):
        class_name = dictionary.get(_CLASS_MARKER, None)
        if class_name == u"FilePath":
            return FilePath(dictionary.get(u"path").encode("utf-8"))
        elif class_name in classes:
            dictionary = dictionary.copy()
            dictionary.pop(_CLASS_MARKER)
            return classes[class_name].create(dictionary)
        else:
            return dictionary
    return loads(data, object_hook=decode_object)

class ConfigurationPersistenceService(Service):
    """
    Persist configuration to disk, and load it back.

    :ivar Deployment _deployment: The current desired deployment configuration.
    """
    def __init__(self, reactor, path):
        """
        :param reactor: Reactor to use for thread pool.
        :param FilePath path: Directory where desired deployment will be
            persisted.
        """
        self._path = path
        self._change_callbacks = []

    def startService(self):
        if not self._path.exists():
            self._path.makedirs()
        self._config_path = self._path.child(b"current_configuration.pickle")
        if self._config_path.exists():
            self._deployment = wire_decode(
                self._config_path.getContent())
        else:
            self._deployment = Deployment(nodes=frozenset())
            self._sync_save(self._deployment)

    def register(self, change_callback):
        """
        Register a function to be called whenever the configuration changes.

        :param change_callback: Callable that takes no arguments, will be
            called when configuration changes.
        """
        self._change_callbacks.append(change_callback)

    def _sync_save(self, deployment):
        """
        Save and flush new deployment to disk synchronously.
        """
        self._config_path.setContent(wire_encode(deployment))

    def save(self, deployment):
        """
        Save and flush new deployment to disk.

        :return Deferred: Fires when write is finished.
        """
        self._sync_save(deployment)
        self._deployment = deployment
        # At some future point this will likely involve talking to a
        # distributed system (e.g. ZooKeeper or etcd), so the API doesn't
        # guarantee immediate saving of the data.
        for callback in self._change_callbacks:
            # Handle errors by catching and logging them
            # https://clusterhq.atlassian.net/browse/FLOC-1311
            callback()
        return succeed(None)

    def get(self):
        """
        Retrieve current configuration.

        It should not be mutated.

        :return Deployment: The current desired configuration.
        """
        return self._deployment
<commit_msg>Address review comment: Better file name.<commit_after># Copyright Hybrid Logic Ltd.  See LICENSE file for details.

"""
Persistence of cluster configuration.
"""

from pyrsistent import PRecord, PVector, PMap, PSet
from json import dumps, loads, JSONEncoder

from twisted.python.filepath import FilePath
from twisted.application.service import Service
from twisted.internet.defer import succeed

from ._model import SERIALIZABLE_CLASSES, Deployment

# Serialization marker storing the class name:
_CLASS_MARKER = u"$__class__$"

class _ConfigurationEncoder(JSONEncoder):
    """
    JSON encoder that can encode the configuration model.
    """
    def default(self, obj):
        if isinstance(obj, PRecord):
            result = dict(obj)
            result[_CLASS_MARKER] = obj.__class__.__name__
            return result
        elif isinstance(obj, PMap):
            return dict(obj)
        elif isinstance(obj, (PSet, PVector, set)):
            return list(obj)
        elif isinstance(obj, FilePath):
            return {_CLASS_MARKER: u"FilePath",
                    u"path": obj.path.decode("utf-8")}
        return JSONEncoder.default(self, obj)

def wire_encode(obj):
    """
    Encode the given configuration object into bytes.

    :param obj: An object from the configuration model, e.g. ``Deployment``.
    :return bytes: Encoded object.
    """
    return dumps(obj, cls=_ConfigurationEncoder)

def wire_decode(data):
    """
    Decode the given configuration object from bytes.

    :param bytes data: Encoded object.
    :param obj: An object from the configuration model, e.g. ``Deployment``.
    """
    classes = {cls.__name__: cls for cls in SERIALIZABLE_CLASSES}

    def decode_object(dictionary):
        class_name = dictionary.get(_CLASS_MARKER, None)
        if class_name == u"FilePath":
            return FilePath(dictionary.get(u"path").encode("utf-8"))
        elif class_name in classes:
            dictionary = dictionary.copy()
            dictionary.pop(_CLASS_MARKER)
            return classes[class_name].create(dictionary)
        else:
            return dictionary
    return loads(data, object_hook=decode_object)

class ConfigurationPersistenceService(Service):
    """
    Persist configuration to disk, and load it back.

    :ivar Deployment _deployment: The current desired deployment configuration.
    """
    def __init__(self, reactor, path):
        """
        :param reactor: Reactor to use for thread pool.
        :param FilePath path: Directory where desired deployment will be
            persisted.
        """
        self._path = path
        self._change_callbacks = []

    def startService(self):
        if not self._path.exists():
            self._path.makedirs()
        self._config_path = self._path.child(b"current_configuration.v1.json")
        if self._config_path.exists():
            self._deployment = wire_decode(
                self._config_path.getContent())
        else:
            self._deployment = Deployment(nodes=frozenset())
            self._sync_save(self._deployment)

    def register(self, change_callback):
        """
        Register a function to be called whenever the configuration changes.

        :param change_callback: Callable that takes no arguments, will be
            called when configuration changes.
        """
        self._change_callbacks.append(change_callback)

    def _sync_save(self, deployment):
        """
        Save and flush new deployment to disk synchronously.
        """
        self._config_path.setContent(wire_encode(deployment))

    def save(self, deployment):
        """
        Save and flush new deployment to disk.

        :return Deferred: Fires when write is finished.
        """
        self._sync_save(deployment)
        self._deployment = deployment
        # At some future point this will likely involve talking to a
        # distributed system (e.g. ZooKeeper or etcd), so the API doesn't
        # guarantee immediate saving of the data.
        for callback in self._change_callbacks:
            # Handle errors by catching and logging them
            # https://clusterhq.atlassian.net/browse/FLOC-1311
            callback()
        return succeed(None)

    def get(self):
        """
        Retrieve current configuration.

        It should not be mutated.

        :return Deployment: The current desired configuration.
        """
        return self._deployment
<|endoftext|>


<commit_before>import os
from test.testcases import LiveTornadoTestCase
from test.selenium_helper import SeleniumHelper
from selenium.webdriver.common.keys import Keys

from django.conf import settings

class BibliographyOverviewTest(LiveTornadoTestCase, SeleniumHelper):

    @classmethod
    def setUpClass(cls):
        super(BibliographyOverviewTest, cls).setUpClass()
        cls.base_url = cls.live_server_url
        driver_data = cls.get_drivers(1)
        cls.driver = driver_data["drivers"][0]
        cls.client = driver_data["clients"][0]
        cls.driver.implicitly_wait(driver_data["wait_time"])

    @classmethod
    def tearDownClass(cls):
        cls.driver.quit()
        super(BibliographyOverviewTest, cls).tearDownClass()

    def setUp(self):
        self.verificationErrors = []
        self.accept_next_alert = True
        self.user = self.create_user(
            username='Yeti',
            email='yeti@snowman.com',
            passtext='otter1'
        )
        self.login_user(self.user, self.driver, self.client)

    def test_overview(self):
        driver = self.driver
        driver.get(self.base_url + "/bibliography/")
        driver.find_element_by_xpath(
            "(.//*[normalize-space(text()) and normalize-space(.)="
            "'All categories'])[1]/following::button[1]"
        ).click()
        driver.find_element_by_xpath(
            "(.//*[normalize-space(text()) and normalize-space(.)="
            "'Edit Categories'])[1]/following::input[1]"
        ).click()
        driver.find_element_by_xpath(
            "(.//*[normalize-space(text()) and normalize-space(.)="
            "'Edit Categories'])[1]/following::input[1]"
        ).clear()
        driver.find_element_by_xpath(
            "(.//*[normalize-space(text()) and normalize-space(.)="
            "'Edit Categories'])[1]/following::input[1]"
        ).send_keys("Fish")
        driver.find_element_by_xpath(
            "(.//*[normalize-space(text()) and normalize-space(.)="
            "'Edit Categories'])[1]/following::span[3]"
        ).click()
        driver.find_element_by_xpath(
            "(.//*[normalize-space(text()) and normalize-space(.)="
            "'Edit Categories'])[1]/following::input[2]"
        ).click()
        driver.find_element_by_xpath(
            "(.//*[normalize-space(text()) and normalize-space(.)="
            "'Edit Categories'])[1]/following::input[2]"
        ).clear()
        driver.find_element_by_xpath(
            "(.//*[normalize-space(text()) and normalize-space(.)="
            "'Edit Categories'])[1]/following::input[2]"
        ).send_keys("Table")
        driver.find_element_by_xpath(
            "(.//*[normalize-space(text()) and normalize-space(.)="
            "'Edit Categories'])[1]/following::span[4]"
        ).click()
        driver.find_element_by_xpath(
            "(.//*[normalize-space(text()) and normalize-space(.)="
            "'Edit Categories'])[1]/following::input[3]"
        ).click()
        driver.find_element_by_xpath(
            "(.//*[normalize-space(text()) and normalize-space(.)="
            "'Edit Categories'])[1]/following::input[3]"
        ).clear()
        driver.find_element_by_xpath(
            "(.//*[normalize-space(text()) and normalize-space(.)="
            "'Edit Categories'])[1]/following::input[3]"
        ).send_keys("Jungle")
        driver.find_element_by_xpath(
            "(.//*[normalize-space(text()) and normalize-space(.)="
            "'Edit Categories'])[1]/following::button[2]"
        ).click()
        driver.find_element_by_xpath(
            "(.//*[normalize-space(text()) and normalize-space(.)="
            "'All categories'])[1]/following::button[1]"
        ).click()
        try:
            self.assertEqual("Fish", driver.find_element_by_id(
                "categoryTitle_1").get_attribute("value"))
        except AssertionError as e:
            self.verificationErrors.append(str(e))
        try:
            self.assertEqual("Table", driver.find_element_by_id(
                "categoryTitle_2").get_attribute("value"))
        except AssertionError as e:
            self.verificationErrors.append(str(e))
        driver.find_element_by_xpath(
            "(.//*[normalize-space(text()) and normalize-space(.)="
            "'Edit Categories'])[1]/following::button[2]"
        ).click()
        driver.find_element_by_xpath(
            "(.//*[normalize-space(text()) and normalize-space(.)="
            "'Edit categories'])[1]/following::button[1]"
        ).click()
        driver.find_element_by_id("select-bibtype").click()
        driver.find_element_by_css_selector(
            "#select-bibtype option[value=article]").click()
        title_of_publication = driver.find_element_by_css_selector(
            ".journaltitle .ProseMirror")
        title_of_publication.click()
        title_of_publication.send_keys("Title of publication")
        title = driver.find_element_by_css_selector(".title .ProseMirror")
        title.click()
        title.send_keys("The title")
        first_name = driver.find_element_by_css_selector(".given .ProseMirror")
        first_name.click()
        first_name.click()  # TODO: We should not need two clicks
        first_name.send_keys("Hans")
        last_name = driver.find_element_by_css_selector(".family .ProseMirror")
        last_name.click()
        last_name.send_keys("Hansen")
        date_field = driver.find_element_by_css_selector("input.date")
        date_field.click()
        date_field.send_keys("1984")
        driver.find_element_by_link_text("Categories").click()
        driver.find_element_by_xpath(
            "(.//*[normalize-space(text()) and normalize-space(.)="
            "'Categories'])[2]/following::div[1]"
        ).click()
        driver.find_element_by_xpath(
            "(.//*[normalize-space(text()) and normalize-space(.)="
            "'Table'])[1]/following::div[1]"
        ).click()
        try:
            self.assertEqual("Jungle", driver.find_element_by_xpath(
                "(.//*[normalize-space(text()) and normalize-space(.)="
                "'Table'])[1]/following::div[1]"
            ).text)
        except AssertionError as e:
            self.verificationErrors.append(str(e))
        driver.find_element_by_xpath(
            "(.//*[normalize-space(text()) and normalize-space(.)="
            "'CasE ProTecT'])[1]/following::button[1]"
        ).click()
        # Closed citation dialog
        search_input = driver.find_element_by_css_selector(
            ".fw-overview-menu-item .fw-button input[type=text]")
        search_input.click()
        search_input.send_keys("women")
        try:
            self.assertEqual(
                "No sources registered",
                driver.find_element_by_css_selector(
                    ".fw-document-table.fw-large.dataTable-table td"
                ).text
            )
        except AssertionError as e:
            self.verificationErrors.append(str(e))
        search_input.send_keys(Keys.BACKSPACE)
        search_input.send_keys(Keys.BACKSPACE)
        search_input.send_keys(Keys.BACKSPACE)
        search_input.send_keys(Keys.BACKSPACE)
        search_input.send_keys(Keys.BACKSPACE)
        try:
            self.assertEqual(
                "The title",
                driver.find_element_by_css_selector(
                    ".fw-document-table-title .edit-bib").text
            )
        except AssertionError as e:
            self.verificationErrors.append(str(e))
        driver.find_element_by_css_selector(
            "button[title='Upload BibTeX file']").click()
        # bibliography path
        bib_path = os.path.join(
            settings.PROJECT_PATH,
            'bibliography/tests/uploads/bibliography.bib'
        )
        driver.find_element_by_id("bib-uploader").send_keys(bib_path)
        driver.find_element_by_xpath(
            "(.//*[normalize-space(text()) and normalize-space(.)="
            "'bibliography.bib'])[1]/following::button[1]"
        ).click()
        book_title_el = WebDriverWait(driver, self.wait_time).until(
            EC.presence_of_element_located(
                (
                    By.CSS_SELECTOR,
                    ".fw-document-table tr:nth-child(2) .edit-bib"
                )
            )
        )
        self.assertEqual(
            "Lean UX: Applying lean principles to improve user experience",
            driver.find_element_by_css_selector(
                ".fw-document-table tr:nth-child(2) .edit-bib").text
        )

    def tearDown(self):
        self.driver.quit()
        self.assertEqual([], self.verificationErrors)
<commit_msg>update bibliography.tests.test_overview.py<commit_after># -*- coding: utf-8 -*-
import os
from test.testcases import LiveTornadoTestCase
from test.selenium_helper import SeleniumHelper
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC

from django.conf import settings

class BibliographyOverviewTest(LiveTornadoTestCase, SeleniumHelper):

    @classmethod
    def setUpClass(cls):
        super(BibliographyOverviewTest, cls).setUpClass()
        cls.base_url = cls.live_server_url
        driver_data = cls.get_drivers(1)
        cls.driver = driver_data["drivers"][0]
        cls.client = driver_data["clients"][0]
        cls.driver.implicitly_wait(driver_data["wait_time"])
        cls.wait_time = driver_data["wait_time"]

    @classmethod
    def tearDownClass(cls):
        cls.driver.quit()
        super(BibliographyOverviewTest, cls).tearDownClass()

    def setUp(self):
        self.verificationErrors = []
        self.accept_next_alert = True
        self.user = self.create_user(
            username='Yeti',
            email='yeti@snowman.com',
            passtext='otter1'
        )
        self.login_user(self.user, self.driver, self.client)

    def test_overview(self):
        driver = self.driver
        driver.get(self.base_url + "/bibliography/")
        driver.find_element_by_xpath(
            "(.//*[normalize-space(text()) and normalize-space(.)="
            "'All categories'])[1]/following::button[1]"
        ).click()
        driver.find_element_by_xpath(
            "(.//*[normalize-space(text()) and normalize-space(.)="
            "'Edit Categories'])[1]/following::input[1]"
        ).click()
        driver.find_element_by_xpath(
            "(.//*[normalize-space(text()) and normalize-space(.)="
            "'Edit Categories'])[1]/following::input[1]"
        ).clear()
        driver.find_element_by_xpath(
            "(.//*[normalize-space(text()) and normalize-space(.)="
            "'Edit Categories'])[1]/following::input[1]"
        ).send_keys("Fish")
        driver.find_element_by_xpath(
            "(.//*[normalize-space(text()) and normalize-space(.)="
            "'Edit Categories'])[1]/following::span[3]"
        ).click()
        driver.find_element_by_xpath(
            "(.//*[normalize-space(text()) and normalize-space(.)="
            "'Edit Categories'])[1]/following::input[2]"
        ).click()
        driver.find_element_by_xpath(
            "(.//*[normalize-space(text()) and normalize-space(.)="
            "'Edit Categories'])[1]/following::input[2]"
        ).clear()
        driver.find_element_by_xpath(
            "(.//*[normalize-space(text()) and normalize-space(.)="
            "'Edit Categories'])[1]/following::input[2]"
        ).send_keys("Table")
        driver.find_element_by_xpath(
            "(.//*[normalize-space(text()) and normalize-space(.)="
            "'Edit Categories'])[1]/following::span[4]"
        ).click()
        driver.find_element_by_xpath(
            "(.//*[normalize-space(text()) and normalize-space(.)="
            "'Edit Categories'])[1]/following::input[3]"
        ).click()
        driver.find_element_by_xpath(
            "(.//*[normalize-space(text()) and normalize-space(.)="
            "'Edit Categories'])[1]/following::input[3]"
        ).clear()
        driver.find_element_by_xpath(
            "(.//*[normalize-space(text()) and normalize-space(.)="
            "'Edit Categories'])[1]/following::input[3]"
        ).send_keys("Jungle")
        driver.find_element_by_xpath(
            "(.//*[normalize-space(text()) and normalize-space(.)="
            "'Edit Categories'])[1]/following::button[2]"
        ).click()
        driver.find_element_by_xpath(
            "(.//*[normalize-space(text()) and normalize-space(.)="
            "'All categories'])[1]/following::button[1]"
        ).click()
        try:
            self.assertEqual("Fish", driver.find_element_by_id(
                "categoryTitle_1").get_attribute("value"))
        except AssertionError as e:
            self.verificationErrors.append(str(e))
        try:
            self.assertEqual("Table", driver.find_element_by_id(
                "categoryTitle_2").get_attribute("value"))
        except AssertionError as e:
            self.verificationErrors.append(str(e))
        driver.find_element_by_xpath(
            "(.//*[normalize-space(text()) and normalize-space(.)="
            "'Edit Categories'])[1]/following::button[2]"
        ).click()
        driver.find_element_by_xpath(
            "(.//*[normalize-space(text()) and normalize-space(.)="
            "'Edit categories'])[1]/following::button[1]"
        ).click()
        driver.find_element_by_id("select-bibtype").click()
        driver.find_element_by_css_selector(
            "#select-bibtype option[value=article]").click()
        title_of_publication = driver.find_element_by_css_selector(
            ".journaltitle .ProseMirror")
        title_of_publication.click()
        title_of_publication.send_keys("Title of publication")
        title = driver.find_element_by_css_selector(".title .ProseMirror")
        title.click()
        title.send_keys("The title")
        first_name = driver.find_element_by_css_selector(".given .ProseMirror")
        first_name.click()
        first_name.click()  # TODO: We should not need two clicks
        first_name.send_keys("Hans")
        last_name = driver.find_element_by_css_selector(".family .ProseMirror")
        last_name.click()
        last_name.send_keys("Hansen")
        date_field = driver.find_element_by_css_selector("input.date")
        date_field.click()
        date_field.send_keys("1984")
        driver.find_element_by_link_text("Categories").click()
        driver.find_element_by_xpath(
            "(.//*[normalize-space(text()) and normalize-space(.)="
            "'Categories'])[2]/following::div[1]"
        ).click()
        driver.find_element_by_xpath(
            "(.//*[normalize-space(text()) and normalize-space(.)="
            "'Table'])[1]/following::div[1]"
        ).click()
        try:
            self.assertEqual("Jungle", driver.find_element_by_xpath(
                "(.//*[normalize-space(text()) and normalize-space(.)="
                "'Table'])[1]/following::div[1]"
            ).text)
        except AssertionError as e:
            self.verificationErrors.append(str(e))
        driver.find_element_by_xpath(
            "(.//*[normalize-space(text()) and normalize-space(.)="
            "'CasE ProTecT'])[1]/following::button[1]"
        ).click()
        # Closed citation dialog
        search_input = driver.find_element_by_css_selector(
            ".fw-overview-menu-item .fw-button input[type=text]")
        search_input.click()
        search_input.send_keys("women")
        try:
            self.assertEqual(
                "No sources registered",
                driver.find_element_by_css_selector(
                    ".fw-document-table.fw-large.dataTable-table td"
                ).text
            )
        except AssertionError as e:
            self.verificationErrors.append(str(e))
        search_input.send_keys(Keys.BACKSPACE)
        search_input.send_keys(Keys.BACKSPACE)
        search_input.send_keys(Keys.BACKSPACE)
        search_input.send_keys(Keys.BACKSPACE)
        search_input.send_keys(Keys.BACKSPACE)
        try:
            self.assertEqual(
                "The title",
                driver.find_element_by_css_selector(
                    ".fw-document-table-title .edit-bib").text
            )
        except AssertionError as e:
            self.verificationErrors.append(str(e))
        driver.find_element_by_css_selector(
            "button[title='Upload BibTeX file']").click()
        # bibliography path
        bib_path = os.path.join(
            settings.PROJECT_PATH,
            'bibliography/tests/uploads/bibliography.bib'
        )
        driver.find_element_by_id("bib-uploader").send_keys(bib_path)
        driver.find_element_by_xpath(
            "(.//*[normalize-space(text()) and normalize-space(.)="
            "'bibliography.bib'])[1]/following::button[1]"
        ).click()
        book_title_el = WebDriverWait(driver, self.wait_time).until(
            EC.presence_of_element_located(
                (
                    By.CSS_SELECTOR,
                    ".fw-document-table tr:nth-child(2) .edit-bib"
                )
            )
        )
        self.assertEqual(
            'Lean UX: Applying lean principles to improve user experience',
            book_title_el.text
        )

    def tearDown(self):
        self.driver.quit()
        self.assertEqual([], self.verificationErrors)
<|endoftext|>


<commit_before>import time
import os
import urlparse, urllib
import sys
sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
from nw_util import *

def path2url(path):
        return urlparse.urljoin(
                  'file:', urllib.pathname2url(path))

from selenium import webdriver
from selenium.webdriver.chrome.options import Options

testdir = os.path.dirname(os.path.abspath(__file__))
os.chdir(testdir)

chrome_options = Options()
chrome_options.add_argument("nwapp=" + testdir)
chrome_options.add_experimental_option("windowTypes", ["webview"])

capabilities = {"pageLoadStrategy": "none"}

driver = webdriver.Chrome(executable_path=os.environ['CHROMEDRIVER'], chrome_options=chrome_options, desired_capabilities = capabilities, service_log_path="log", service_args=["--verbose"])
driver.implicitly_wait(5)
try:
    print driver.current_url
    #driver.find_element_by_id('testbtn').click()
    timeout = 10
    found = False
    while timeout > 0 and not found:
        for handle in driver.window_handles:
            driver.switch_to_window(handle)
            if driver.current_url.startswith('devtools://'):
                found = True
                break
        timeout = timeout - 1
        time.sleep(1)
    assert(found)
finally:
    driver.quit()
<commit_msg>[test] unflacky webview-cdt<commit_after>import time
import os
import urlparse, urllib
import sys
sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
from nw_util import *

def path2url(path):
        return urlparse.urljoin(
                  'file:', urllib.pathname2url(path))

from selenium import webdriver
from selenium.webdriver.chrome.options import Options

testdir = os.path.dirname(os.path.abspath(__file__))
os.chdir(testdir)

chrome_options = Options()
chrome_options.add_argument("nwapp=" + testdir)
chrome_options.add_experimental_option("windowTypes", ["webview"])

capabilities = {"pageLoadStrategy": "none"}

driver = webdriver.Chrome(executable_path=os.environ['CHROMEDRIVER'], chrome_options=chrome_options, desired_capabilities = capabilities, service_log_path="log", service_args=["--verbose"])
driver.implicitly_wait(5)
try:
    print driver.current_url
    #driver.find_element_by_id('testbtn').click()
    timeout = 15
    found = False
    while timeout > 0 and not found:
        try:
            for handle in driver.window_handles:
                driver.switch_to_window(handle)
                if driver.current_url.startswith('devtools://'):
                    found = True
                    break
        except:
            pass
        timeout = timeout - 1
        time.sleep(1)
    assert(found)
finally:
    driver.quit()
<|endoftext|>


<commit_before>""" RHEAS analysis module.

.. module:: analysis
   :synopsis: Module for post-processing RHEAS outputs

.. moduleauthor:: Kostas Andreadis <kandread@jpl.nasa.gov>

"""

import dbio
import logging
from osgeo import ogr
from datetime import datetime

def cropYield(shapefile, name, startdate="", enddate="", crop="maize", dbname="rheas"):
    """Extract crop yield from a specified simulation *name* for dates ranging
    from *startdate* to *enddate*, and saves them a *shapefile*."""
    logging.basicConfig(level=logging.INFO, format='%(message)s')
    log = logging.getLogger(__name__)
    db = dbio.connect(dbname)
    cur = db.cursor()
    datesql = ""
    if len(startdate) > 0:
        try:
            sdt = datetime.strptime(startdate, "%Y-%m-%d")
            datesql = "and fdate>=date'{0}'".format(sdt.strftime("%Y-%m-%d"))
        except:
            log.warning("Start date is invalid and will be ignored.")
    if len(enddate) > 0:
        try:
            edt = datetime.strptime(enddate, "%Y-%m-%d")
            datesql += "and fdate<=date'{0}'".format(edt.strftime("%y-%m-%d"))
        except:
            log.warning("End date is invalid and will be ignored.")
    fsql = "with f as (select gid,geom,gwad,ensemble,fdate from (select gid,geom,gwad,ensemble,fdate,row_number() over (partition by gid,ensemble order by gwad desc) as rn from {0}.dssat) gwadtable where rn=1 {1})".format(name, datesql)
    sql = "{0} select gid,st_astext(geom),max(gwad) as max_yield,avg(gwad) as avg_yield,stddev(gwad) as std_yield,max(fdate) as fdate from f group by gid,geom".format(fsql)
    cur.execute(sql)
    if bool(cur.rowcount):
        results = cur.fetchall()
        drv = ogr.GetDriverByName("ESRI Shapefile")
        ds = drv.CreateDataSource(shapefile)
        lyr = ds.CreateLayer("yield", geom_type=ogr.wkbMultiPolygon)
        lyr.CreateField(ogr.FieldDefn("gid", ogr.OFTInteger))
        lyr.CreateField(ogr.FieldDefn("average", ogr.OFTReal))
        lyr.CreateField(ogr.FieldDefn("maximum", ogr.OFTReal))
        lyr.CreateField(ogr.FieldDefn("minimum", ogr.OFTReal))
        for row in results:
            feat = ogr.Feature(lyr.GetLayerDefn())
            feat.SetField("gid", row[0])
            feat.SetField("maximum", row[2])
            feat.SetField("average", row[3])
            feat.SetField("minimum", row[4])
            feat.SetGeometry(ogr.CreateGeometryFromWkt(row[1]))
            lyr.CreateFeature(feat)
            feat.Destroy()
        ds.Destroy()
<commit_msg>Save raster files from simulation stored in the database.<commit_after>""" RHEAS analysis module.

.. module:: analysis
   :synopsis: Module for post-processing RHEAS outputs

.. moduleauthor:: Kostas Andreadis <kandread@jpl.nasa.gov>

"""

import dbio
import logging
from osgeo import ogr
from datetime import datetime

def cropYield(shapefile, name, startdate="", enddate="", crop="maize", dbname="rheas"):
    """Extract crop yield from a specified simulation *name* for dates ranging
    from *startdate* to *enddate*, and saves them a *shapefile*."""
    logging.basicConfig(level=logging.INFO, format='%(message)s')
    log = logging.getLogger(__name__)
    db = dbio.connect(dbname)
    cur = db.cursor()
    datesql = ""
    if len(startdate) > 0:
        try:
            sdt = datetime.strptime(startdate, "%Y-%m-%d")
            datesql = "and fdate>=date'{0}'".format(sdt.strftime("%Y-%m-%d"))
        except ValueError:
            log.warning("Start date is invalid and will be ignored.")
    if len(enddate) > 0:
        try:
            edt = datetime.strptime(enddate, "%Y-%m-%d")
            datesql += "and fdate<=date'{0}'".format(edt.strftime("%y-%m-%d"))
        except ValueError:
            log.warning("End date is invalid and will be ignored.")
    fsql = "with f as (select gid,geom,gwad,ensemble,fdate from (select gid,geom,gwad,ensemble,fdate,row_number() over (partition by gid,ensemble order by gwad desc) as rn from {0}.dssat) gwadtable where rn=1 {1})".format(name, datesql)
    sql = "{0} select gid,st_astext(geom),max(gwad) as max_yield,avg(gwad) as avg_yield,stddev(gwad) as std_yield,max(fdate) as fdate from f group by gid,geom".format(fsql)
    cur.execute(sql)
    if bool(cur.rowcount):
        results = cur.fetchall()
        drv = ogr.GetDriverByName("ESRI Shapefile")
        ds = drv.CreateDataSource(shapefile)
        lyr = ds.CreateLayer("yield", geom_type=ogr.wkbMultiPolygon)
        lyr.CreateField(ogr.FieldDefn("gid", ogr.OFTInteger))
        lyr.CreateField(ogr.FieldDefn("average", ogr.OFTReal))
        lyr.CreateField(ogr.FieldDefn("maximum", ogr.OFTReal))
        lyr.CreateField(ogr.FieldDefn("minimum", ogr.OFTReal))
        for row in results:
            feat = ogr.Feature(lyr.GetLayerDefn())
            feat.SetField("gid", row[0])
            feat.SetField("maximum", row[2])
            feat.SetField("average", row[3])
            feat.SetField("minimum", row[4])
            feat.SetGeometry(ogr.CreateGeometryFromWkt(row[1]))
            lyr.CreateFeature(feat)
            feat.Destroy()
        ds.Destroy()

def saveVariable(filepath, name, varname, startdate="", enddate="", dbname="rheas"):
    """Extracts geophysical variable *varname* from simulation *name* for date range between
    *startdate* and *enddate*, and either saves the rasters if *filepath* is a
    directory, or as a CSV file with each column associated to a polygon of the
    *filepath* shapefile."""
    if filepath.endswith(".shp"):
        pass
    else:
        _saveRasters(filepath, name, varname, startdate, enddate, dbname)

def _saveRasters(filepath, name, varname, startdate, enddate, dbname):
    """"Save geophysical variable from *dbname* database, between *startdate*
    and *enddate* dates into Geotif files inside *filepath* directory."""
    logging.basicConfig(level=logging.INFO, format='%(message)s')
    log = logging.getLogger(__name__)
    if dbio.tableExists(dbname, name, varname):
        db = dbio.connect(dbname)
        cur = db.cursor()
        sql = "select fdate,st_astiff(st_union(rast)) as tif from {0}.{1}".format(name, varname)
        try:
            sdt = datetime.strptime(startdate, "%Y-%m-%d")
            edt = datetime.strptime(enddate, "%Y-%m-%d")
            sql += " where fdate>=date'{0}' and fdate<=date'{1} group by fdate".format(sdt.strftime("%Y-%m-%d"), edt.strftime("%Y-%m-%d"))
        except ValueError:
            sql += " group by fdate"
            log.warning("Start and/or end dates were invalid. Ignoring...")
        cur.execute(sql)
        results = cur.fetchall()
        for res in results:
            with open("{0}/{1}_{2}.tif".format(filepath, varname, res[0].strftime("%Y%m%d")), 'wb') as fout:
                fout.write(res[1])
    else:
        log.error("Variable {0} does not exist in schema {1}.".format(varname, name))
<|endoftext|>


<commit_before><commit_msg>add imports<commit_after>from .query import QueryBase, RawQuery, QueryGroup<|endoftext|>