<commit_before>from __future__ import division
import os
import glob
import string
import random
import numpy as np
from scipy.odr import *
import json
import matplotlib.pyplot as plt
import CoolProp
import CoolProp.CoolProp as CP

LIBRARY = [i/6.0 for i in range(1,151)]+[0.35+i/2000 for i in range(1,100)]+[0.05+0.001*i for i in range(1,100)]+[i+0.5 for i in range(10)]
#LIBRARY = [i/1000 for i in range(1,20000)]

class Sample(object):
    def __init__(self,v):
        self.v = v

class GeneticAncillaryFitter(object):
    def __init__(self,
               num_samples = 600, # Have this many chromos in the sample group
               num_selected = 60, # Have this many chromos in the selected group
               mutation_factor = 2, # Randomly mutate 1/n of the chromosomes
               num_powers = 5, # How many powers in the fit
               Ref = 'R407C',
               value = 'rhoV',
               addTr = True,
               values = None,
               Tlims = None
                ):
        self.num_samples = num_samples
        self.num_selected = num_selected
        self.mutation_factor = mutation_factor
        self.num_powers = num_powers
        self.addTr = addTr
        self.value = value
        self.Ref = Ref

        #Thermodynamics
        from CoolProp.CoolProp import PropsSI

        if values is None:
            self.Tc = PropsSI(Ref,'Tcrit')
            self.pc = PropsSI(Ref,'pcrit')
            self.rhoc = PropsSI(Ref,'rhomolar_critical')
            self.Tmin = PropsSI(Ref,'Tmin')
            if Tlims is None:
                self.T = np.append(np.linspace(self.Tmin+1e-14, self.Tc-1,150), np.logspace(np.log10(self.Tc-1), np.log10(self.Tc)-1e-15,40))
            else:
                self.T = np.linspace(Tlims[0],Tlims[1])
            self.pL = np.array(PropsSI('P','T',self.T,'Q',[0]*len(self.T),Ref))
            self.pV = np.array(PropsSI('P','T',self.T,'Q',[1]*len(self.T),Ref))
            self.rhoL = PropsSI('D','T',self.T,'Q',[0]*len(self.T),Ref)
            self.rhoV = PropsSI('D','T',self.T,'Q',[1]*len(self.T),Ref)
        else:
            self.Tc = values['Tcrit']
            self.pc = values['pcrit']
            self.rhoc = values['rhocrit']
            self.Tmin = values['Tmin']
            self.T = np.array(values['T'])
            self.p = np.array(values['p'])
            self.pL = np.array(values['p'])
            self.pV = np.array(values['p'])
            self.rhoL = np.array(values['rhoL'])
            self.rhoV = np.array(values['rhoV'])

        self.logpLpc = (np.log(self.pL)-np.log(self.pc))
        self.logpVpc = (np.log(self.pV)-np.log(self.pc))
        self.rhoLrhoc = np.array(self.rhoL)/self.rhoc
        self.rhoVrhoc = np.array(self.rhoV)/self.rhoc
        self.logrhoLrhoc = np.log(self.rhoL)-np.log(self.rhoc)
        self.logrhoVrhoc = np.log(self.rhoV)-np.log(self.rhoc)

        self.x = 1.0-self.T/self.Tc

        MM = PropsSI(Ref,'molemass')
        self.T_r = self.Tc

        if self.value == 'pL':
            self.LHS = self.logpLpc.copy()
            self.EOS_value = self.pL.copy()
            if self.addTr == False:
                self.description = "p' = pc*exp(sum(n_i*theta^t_i))"
            else:
                self.description = "p' = pc*exp(Tc/T*sum(n_i*theta^t_i))"
            self.reducing_value = self.pc
        elif self.value == 'pV':
            self.LHS = self.logpVpc.copy()
            self.EOS_value = self.pV.copy()
            if self.addTr == False:
                self.description = "p'' = pc*exp(sum(n_i*theta^t_i))"
            else:
                self.description = "p'' = pc*exp(Tc/T*sum(n_i*theta^t_i))"
            self.reducing_value = self.pc
        elif self.value == 'rhoL':
            self.LHS = self.logrhoLrhoc.copy()
            self.EOS_value = self.rhoL
            if self.addTr == False:
                self.description = "rho' = rhoc*exp(sum(n_i*theta^t_i))"
            else:
                self.description = "rho' = rhoc*exp(Tc/T*sum(n_i*theta^t_i))"
            self.reducing_value = self.rhoc
        elif self.value == 'rhoV':
            self.LHS = self.logrhoVrhoc.copy()
            self.EOS_value = self.rhoV
            if self.addTr == False:
                self.description = "rho'' = rhoc*exp(sum(n_i*theta^t_i))"
            else:
                self.description = "rho'' = rhoc*exp(Tc/T*sum(n_i*theta^t_i))"
            self.reducing_value = self.rhoc
        elif self.value == 'rhoLnoexp':
            self.LHS = (self.rhoLrhoc-1).copy()
            self.EOS_value = self.rhoL
            self.description = "rho' = rhoc*(1+sum(n_i*theta^t_i))"
            self.reducing_value = self.rhoc
        else:
            raise ValueError

        if self.value == 'rhoLnoexp' and self.addTr:
            raise ValueError('Invalid combination')

        if self.addTr:
            self.LHS *= self.T/self.Tc

    def generate_random_chromosomes(self,):
        '''
        Create a list of random chromosomes to seed our alogrithm.
        '''
        chromos = []
        while len(chromos) < self.num_samples:
            chromos.append(Sample(sorted(random.sample(LIBRARY,self.num_powers))))
        return chromos

    def fitness(self, chromo):
        '''
        Fitness of a chromo is the sum of the squares of the error of the correlation
        '''

        # theta^t where the i-th row is equal to theta^t[i]
        # We need these terms later on to build the A and b matrices
        theta_t = (self.x.reshape(-1, 1)**chromo.v).T

        # TODO: more numpy broadcasting should speed this up even more
        # Another few times faster ought to be possible
        I = len(chromo.v)
        A = np.zeros((I,I))
        b = np.zeros((I,1))
        for i in range(I):
            for j in range(I):
                A[i,j] = np.sum(theta_t[i]*theta_t[j])
            b[i] = np.sum(theta_t[i]*self.LHS)

        # If you end up with a singular matrix, quit this run
        try:
            n = np.linalg.solve(A, b).T
        except np.linalg.linalg.LinAlgError as E:
            chromo.fitness = 1e99
            return

        chromo.beta = n
        RHS = np.sum(n * self.x.reshape(-1, 1)**chromo.v, axis = 1)

        if self.addTr:
            RHS *= self.Tc/self.T

        if self.value in ['pL','pV']:
            fit_value = np.exp(RHS)*self.pc
        elif self.value in ['rhoL','rhoV']:
            fit_value = np.exp(RHS)*self.rhoc
        elif self.value == 'rhoLnoexp':
            fit_value = self.rhoc*(1+RHS)
        else:
            raise ValueError

        max_abserror = np.max(np.abs((fit_value/self.EOS_value)-1)*100)

        chromo.fitness = max_abserror
        chromo.fit_value = fit_value
        chromo.max_abserror = max_abserror

        return chromo.fitness

    def tourny_select_chromo(self, samples):
        '''
        Randomly select two chromosomes from the samples, then return the one
        with the best fitness.
        '''
        a = random.choice(samples)
        b = random.choice(samples)
        if a.fitness < b.fitness:
            return a
        else:
            return b

    def breed(self, a, b):
        '''
        Breed two chromosomes by splicing them in a random spot and combining
        them together to form two new chromos.
        '''
        splice_pos = random.randrange(len(a.v))
        new_a = a.v[:splice_pos] + b.v[splice_pos:]
        new_b = b.v[:splice_pos] + a.v[splice_pos:]
        return Sample(sorted(new_a)), Sample(sorted(new_b))

    def mutate(self, chromo):
        '''
        Mutate a chromosome by changing one of the parameters, but only if it improves the fitness
        '''
        v = chromo.v
        if hasattr(chromo,'fitness'):
            old_fitness = chromo.fitness
        else:
            old_fitness = self.fitness(chromo)

        for i in range(10):
            pos = random.randrange(len(chromo.v))
            chromo.v[pos] = random.choice(LIBRARY)
            new_fitness = self.fitness(chromo)
            if new_fitness < old_fitness:
                return chromo
            else:
                return Sample(sorted(v))

    def run(self):
        # Create a random sample of chromos
        samples = self.generate_random_chromosomes()

        # Calculate the fitness for the initial chromosomes
        for chromo in samples:
            self.fitness(chromo)
#         print '#'

        decorated = [(sample.fitness,sample) for sample in samples]
        decorated.sort()
        samples = [s for sv,s in decorated]
        values = [sv for sv,s in decorated]
        plt.plot(values[0:len(values)//2])
        plt.close()

        # Main loop: each generation select a subset of the sample and breed from
        # them.
        generation = -1
        while generation < 0 or samples[0].fitness > 0.02 or (generation < 3 and generation < 15):
            generation += 1

            # Generate the selected group from sample- take the top 10% of samples
            # and tourny select to generate the rest of selected.
            ten_percent = int(len(samples)*.1)
            selected = samples[:ten_percent]
            while len(selected) < self.num_selected:
                selected.append(self.tourny_select_chromo(samples))

            # Generate the solution group by breeding random chromos from selected
            solution = []
            while len(solution) < self.num_samples:
                solution.extend(self.breed(random.choice(selected),
                                         random.choice(selected)))

            # Apply a mutation to a subset of the solution set
            mutate_indices = random.sample(range(len(solution)), len(solution)//self.mutation_factor)
            for i in mutate_indices:
                solution[i] = self.mutate(solution[i])

            for chromo in solution:
                self.fitness(chromo)
#             print '#'

            decorated = [(sample.fitness,sample) for sample in solution]
            decorated.sort()
            samples = [s for sv,s in decorated]

#             print '------------------  Top 10 values  ---------------'
#             for sample in samples[0:10]:
#                 print sample.v, sample.fitness, sample.max_abserror

#             print '// Max error is ',samples[0].max_abserror,'% between',np.min(self.T),'and',np.max(self.T),'K'
#             print str(samples[0].v), samples[0].beta.tolist()

            # Print useful stats about this generation
            (min, median, max) =  [samples[0].fitness, samples[len(samples)//2].fitness, samples[-1].fitness]
#             print("{0} best value: {1}. fitness: best {2}, median {3}, worst {4}".format(generation, samples[0].v, min, median, max))

            # If the string representations of all the chromosomes are the same, stop
            if len(set([str(s.v) for s in samples[0:5]])) == 1:
                break

        self.fitness(samples[0])

        print self.value
        print '// Max error is ',samples[0].max_abserror,'% between',np.min(self.T),'and',np.max(self.T),'K'
#         print str(samples[0].v), samples[0].beta.tolist()

        j = dict()
        j['n'] = samples[0].beta.squeeze().tolist()
        j['t'] = samples[0].v
        j['Tmin'] = np.min(self.T)
        j['Tmax'] = np.max(self.T)
        j['type'] = self.value
        j['using_tau_r'] = self.addTr
        j['reducing_value'] = self.reducing_value
        j['T_r'] = self.Tc

        # Informational, not used
        j['max_abserror_percentage'] = samples[0].max_abserror
        j['description'] = self.description

        return j

def build_ancillaries(name, **kwargs):

    j = dict()
    j['ANCILLARIES'] = dict()
    gaf = GeneticAncillaryFitter(Ref = name, value = 'pL', addTr = True, num_powers = 6, **kwargs)
    j['ANCILLARIES']['pL'] = gaf.run()
    gaf = GeneticAncillaryFitter(Ref = name, value = 'pV', addTr = True, num_powers = 6, **kwargs)
    j['ANCILLARIES']['pV'] = gaf.run()
    gaf = GeneticAncillaryFitter(Ref = name, value = 'rhoLnoexp', addTr = False, num_powers = 6, **kwargs)
    j['ANCILLARIES']['rhoL'] = gaf.run()
    gaf = GeneticAncillaryFitter(Ref = name, value = 'rhoV', addTr = True, num_powers = 6, **kwargs)
    j['ANCILLARIES']['rhoV'] = gaf.run()

    fp = open(os.path.join('ancillaries',name+'_anc.json'),'w')
    print >> fp, json.dumps(j, indent = 2)
    fp.close()

def build_all_ancillaries():
    for fluid in sorted(CoolProp.__fluids__):
        print fluid
        if fluid in ['SES36']:
            build_ancillaries(fluid, Tlims = [CP.Props(fluid,'Ttriple'), CP.Props(fluid, 'Tcrit')-1])
        elif fluid == 'R507A':
            build_ancillaries(fluid, Tlims = [CP.Props(fluid,'Ttriple'), CP.Props(fluid, 'Tcrit')-0.1])
        elif fluid == 'R407F':
            build_ancillaries(fluid, Tlims = [CP.Props(fluid,'Ttriple'), CP.Props(fluid, 'Tcrit')-2])
        else:
            build_ancillaries(fluid)

if __name__ == "__main__":

    fluid = 'R245fa'
    RPfluid = fluid
    build_ancillaries(RPfluid, Tlims = [CP.PropsSI(fluid,'Ttriple'), CP.PropsSI(fluid, 'Tcrit')-0.01])

    #~ build_all_ancillaries()
#     inject_ancillaries()
<commit_msg>Update PVT ancillary fitter to use molar densities<commit_after>from __future__ import division
import os
import glob
import string
import random
import numpy as np
from scipy.odr import *
import json
import matplotlib.pyplot as plt
import CoolProp
import CoolProp.CoolProp as CP

LIBRARY = [i/6.0 for i in range(1,151)]+[0.35+i/2000 for i in range(1,100)]+[0.05+0.001*i for i in range(1,100)]+[i+0.5 for i in range(10)]
#LIBRARY = [i/1000 for i in range(1,20000)]

class Sample(object):
    def __init__(self,v):
        self.v = v

class GeneticAncillaryFitter(object):
    def __init__(self,
               num_samples = 600, # Have this many chromos in the sample group
               num_selected = 60, # Have this many chromos in the selected group
               mutation_factor = 2, # Randomly mutate 1/n of the chromosomes
               num_powers = 5, # How many powers in the fit
               Ref = 'R407C',
               value = 'rhoV',
               addTr = True,
               values = None,
               Tlims = None
                ):
        self.num_samples = num_samples
        self.num_selected = num_selected
        self.mutation_factor = mutation_factor
        self.num_powers = num_powers
        self.addTr = addTr
        self.value = value
        self.Ref = Ref

        #Thermodynamics
        from CoolProp.CoolProp import PropsSI

        if values is None:
            self.Tc = PropsSI(Ref,'Tcrit')
            self.pc = PropsSI(Ref,'pcrit')
            self.rhoc = PropsSI(Ref,'rhomolar_critical')
            self.Tmin = PropsSI(Ref,'Tmin')
            if Tlims is None:
                self.T = np.append(np.linspace(self.Tmin+1e-14, self.Tc-1,150), np.logspace(np.log10(self.Tc-1), np.log10(self.Tc)-1e-15,40))
            else:
                self.T = np.linspace(Tlims[0],Tlims[1])
            self.pL = np.array(PropsSI('P','T',self.T,'Q',[0]*len(self.T),Ref))
            self.pV = np.array(PropsSI('P','T',self.T,'Q',[1]*len(self.T),Ref))
            self.rhoL = PropsSI('Dmolar','T',self.T,'Q',[0]*len(self.T),Ref)
            self.rhoV = PropsSI('Dmolar','T',self.T,'Q',[1]*len(self.T),Ref)
        else:
            self.Tc = values['Tcrit']
            self.pc = values['pcrit']
            self.rhoc = values['rhocrit']
            self.Tmin = values['Tmin']
            self.T = np.array(values['T'])
            self.p = np.array(values['p'])
            self.pL = np.array(values['p'])
            self.pV = np.array(values['p'])
            self.rhoL = np.array(values['rhoL'])
            self.rhoV = np.array(values['rhoV'])

        self.logpLpc = (np.log(self.pL)-np.log(self.pc))
        self.logpVpc = (np.log(self.pV)-np.log(self.pc))
        self.rhoLrhoc = np.array(self.rhoL)/self.rhoc
        self.rhoVrhoc = np.array(self.rhoV)/self.rhoc
        self.logrhoLrhoc = np.log(self.rhoL)-np.log(self.rhoc)
        self.logrhoVrhoc = np.log(self.rhoV)-np.log(self.rhoc)

        self.x = 1.0-self.T/self.Tc

        MM = PropsSI(Ref,'molemass')
        self.T_r = self.Tc

        if self.value == 'pL':
            self.LHS = self.logpLpc.copy()
            self.EOS_value = self.pL.copy()
            if self.addTr == False:
                self.description = "p' = pc*exp(sum(n_i*theta^t_i))"
            else:
                self.description = "p' = pc*exp(Tc/T*sum(n_i*theta^t_i))"
            self.reducing_value = self.pc
        elif self.value == 'pV':
            self.LHS = self.logpVpc.copy()
            self.EOS_value = self.pV.copy()
            if self.addTr == False:
                self.description = "p'' = pc*exp(sum(n_i*theta^t_i))"
            else:
                self.description = "p'' = pc*exp(Tc/T*sum(n_i*theta^t_i))"
            self.reducing_value = self.pc
        elif self.value == 'rhoL':
            self.LHS = self.logrhoLrhoc.copy()
            self.EOS_value = self.rhoL
            if self.addTr == False:
                self.description = "rho' = rhoc*exp(sum(n_i*theta^t_i))"
            else:
                self.description = "rho' = rhoc*exp(Tc/T*sum(n_i*theta^t_i))"
            self.reducing_value = self.rhoc
        elif self.value == 'rhoV':
            self.LHS = self.logrhoVrhoc.copy()
            self.EOS_value = self.rhoV
            if self.addTr == False:
                self.description = "rho'' = rhoc*exp(sum(n_i*theta^t_i))"
            else:
                self.description = "rho'' = rhoc*exp(Tc/T*sum(n_i*theta^t_i))"
            self.reducing_value = self.rhoc
        elif self.value == 'rhoLnoexp':
            self.LHS = (self.rhoLrhoc-1).copy()
            self.EOS_value = self.rhoL
            self.description = "rho' = rhoc*(1+sum(n_i*theta^t_i))"
            self.reducing_value = self.rhoc
        else:
            raise ValueError

        if self.value == 'rhoLnoexp' and self.addTr:
            raise ValueError('Invalid combination')

        if self.addTr:
            self.LHS *= self.T/self.Tc

    def generate_random_chromosomes(self,):
        '''
        Create a list of random chromosomes to seed our alogrithm.
        '''
        chromos = []
        while len(chromos) < self.num_samples:
            chromos.append(Sample(sorted(random.sample(LIBRARY,self.num_powers))))
        return chromos

    def fitness(self, chromo):
        '''
        Fitness of a chromo is the sum of the squares of the error of the correlation
        '''

        # theta^t where the i-th row is equal to theta^t[i]
        # We need these terms later on to build the A and b matrices
        theta_t = (self.x.reshape(-1, 1)**chromo.v).T

        # TODO: more numpy broadcasting should speed this up even more
        # Another few times faster ought to be possible
        I = len(chromo.v)
        A = np.zeros((I,I))
        b = np.zeros((I,1))
        for i in range(I):
            for j in range(I):
                A[i,j] = np.sum(theta_t[i]*theta_t[j])
            b[i] = np.sum(theta_t[i]*self.LHS)

        # If you end up with a singular matrix, quit this run
        try:
            n = np.linalg.solve(A, b).T
        except np.linalg.linalg.LinAlgError as E:
            chromo.fitness = 1e99
            return

        chromo.beta = n
        RHS = np.sum(n * self.x.reshape(-1, 1)**chromo.v, axis = 1)

        if self.addTr:
            RHS *= self.Tc/self.T

        if self.value in ['pL','pV']:
            fit_value = np.exp(RHS)*self.pc
        elif self.value in ['rhoL','rhoV']:
            fit_value = np.exp(RHS)*self.rhoc
        elif self.value == 'rhoLnoexp':
            fit_value = self.rhoc*(1+RHS)
        else:
            raise ValueError

        max_abserror = np.max(np.abs((fit_value/self.EOS_value)-1)*100)

        chromo.fitness = max_abserror
        chromo.fit_value = fit_value
        chromo.max_abserror = max_abserror

        return chromo.fitness

    def tourny_select_chromo(self, samples):
        '''
        Randomly select two chromosomes from the samples, then return the one
        with the best fitness.
        '''
        a = random.choice(samples)
        b = random.choice(samples)
        if a.fitness < b.fitness:
            return a
        else:
            return b

    def breed(self, a, b):
        '''
        Breed two chromosomes by splicing them in a random spot and combining
        them together to form two new chromos.
        '''
        splice_pos = random.randrange(len(a.v))
        new_a = a.v[:splice_pos] + b.v[splice_pos:]
        new_b = b.v[:splice_pos] + a.v[splice_pos:]
        return Sample(sorted(new_a)), Sample(sorted(new_b))

    def mutate(self, chromo):
        '''
        Mutate a chromosome by changing one of the parameters, but only if it improves the fitness
        '''
        v = chromo.v
        if hasattr(chromo,'fitness'):
            old_fitness = chromo.fitness
        else:
            old_fitness = self.fitness(chromo)

        for i in range(10):
            pos = random.randrange(len(chromo.v))
            chromo.v[pos] = random.choice(LIBRARY)
            new_fitness = self.fitness(chromo)
            if new_fitness < old_fitness:
                return chromo
            else:
                return Sample(sorted(v))

    def run(self):
        # Create a random sample of chromos
        samples = self.generate_random_chromosomes()

        # Calculate the fitness for the initial chromosomes
        for chromo in samples:
            self.fitness(chromo)
#         print '#'

        decorated = [(sample.fitness,sample) for sample in samples]
        decorated.sort()
        samples = [s for sv,s in decorated]
        values = [sv for sv,s in decorated]
        plt.plot(values[0:len(values)//2])
        plt.close()

        # Main loop: each generation select a subset of the sample and breed from
        # them.
        generation = -1
        while generation < 0 or samples[0].fitness > 0.02 or (generation < 3 and generation < 15):
            generation += 1

            # Generate the selected group from sample- take the top 10% of samples
            # and tourny select to generate the rest of selected.
            ten_percent = int(len(samples)*.1)
            selected = samples[:ten_percent]
            while len(selected) < self.num_selected:
                selected.append(self.tourny_select_chromo(samples))

            # Generate the solution group by breeding random chromos from selected
            solution = []
            while len(solution) < self.num_samples:
                solution.extend(self.breed(random.choice(selected),
                                         random.choice(selected)))

            # Apply a mutation to a subset of the solution set
            mutate_indices = random.sample(range(len(solution)), len(solution)//self.mutation_factor)
            for i in mutate_indices:
                solution[i] = self.mutate(solution[i])

            for chromo in solution:
                self.fitness(chromo)
#             print '#'

            decorated = [(sample.fitness,sample) for sample in solution]
            decorated.sort()
            samples = [s for sv,s in decorated]

#             print '------------------  Top 10 values  ---------------'
#             for sample in samples[0:10]:
#                 print sample.v, sample.fitness, sample.max_abserror

#             print '// Max error is ',samples[0].max_abserror,'% between',np.min(self.T),'and',np.max(self.T),'K'
#             print str(samples[0].v), samples[0].beta.tolist()

            # Print useful stats about this generation
            (min, median, max) =  [samples[0].fitness, samples[len(samples)//2].fitness, samples[-1].fitness]
#             print("{0} best value: {1}. fitness: best {2}, median {3}, worst {4}".format(generation, samples[0].v, min, median, max))

            # If the string representations of all the chromosomes are the same, stop
            if len(set([str(s.v) for s in samples[0:5]])) == 1:
                break

        self.fitness(samples[0])

        print self.value
        print '// Max error is ',samples[0].max_abserror,'% between',np.min(self.T),'and',np.max(self.T),'K'

        self.fit_value = samples[0].fit_value

        j = dict()
        j['n'] = samples[0].beta.squeeze().tolist()
        j['t'] = samples[0].v
        j['Tmin'] = np.min(self.T)
        j['Tmax'] = np.max(self.T)
        j['type'] = self.value
        j['using_tau_r'] = self.addTr
        j['reducing_value'] = self.reducing_value
        j['T_r'] = self.Tc

        # Informational, not used
        j['max_abserror_percentage'] = samples[0].max_abserror
        j['description'] = self.description

        return j

def build_ancillaries(name, **kwargs):

    j = dict()
    j['ANCILLARIES'] = dict()
    gaf = GeneticAncillaryFitter(Ref = name, value = 'pL', addTr = True, num_powers = 6, **kwargs)
    j['ANCILLARIES']['pL'] = gaf.run()
    gaf = GeneticAncillaryFitter(Ref = name, value = 'pV', addTr = True, num_powers = 6, **kwargs)
    j['ANCILLARIES']['pV'] = gaf.run()
    gaf = GeneticAncillaryFitter(Ref = name, value = 'rhoLnoexp', addTr = False, num_powers = 6, **kwargs)
    j['ANCILLARIES']['rhoL'] = gaf.run()
    gaf = GeneticAncillaryFitter(Ref = name, value = 'rhoV', addTr = True, num_powers = 6, **kwargs)
    j['ANCILLARIES']['rhoV'] = gaf.run()

    fp = open(os.path.join('ancillaries',name+'_anc.json'),'w')
    print >> fp, json.dumps(j, indent = 2)
    fp.close()

def build_all_ancillaries():
    for fluid in sorted(CoolProp.__fluids__):
        print fluid
        if fluid in ['SES36']:
            build_ancillaries(fluid, Tlims = [CP.Props(fluid,'Ttriple'), CP.Props(fluid, 'Tcrit')-1])
        elif fluid == 'R507A':
            build_ancillaries(fluid, Tlims = [CP.Props(fluid,'Ttriple'), CP.Props(fluid, 'Tcrit')-0.1])
        elif fluid == 'R407F':
            build_ancillaries(fluid, Tlims = [CP.Props(fluid,'Ttriple'), CP.Props(fluid, 'Tcrit')-2])
        else:
            build_ancillaries(fluid)

if __name__ == "__main__":

    fluid = 'Methanol'
    RPfluid = fluid
    build_ancillaries(RPfluid, Tlims = [CP.PropsSI(fluid,'Ttriple'), CP.PropsSI(fluid, 'Tcrit')-0.01])

    #~ build_all_ancillaries()
#     inject_ancillaries()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# Em teste
from business_rules import run_all
from business_rules.actions import BaseActions, rule_action
from business_rules.fields import FIELD_NUMERIC, FIELD_TEXT
from moduleOfRules.ActionRules import ActionRules
from moduleOfRules.ConditionsRules import ConditionsRules
from moduleOfRules.Parameters import Parameters
import requests
import json
import os
#metodo que recebe o json que contem regras de contigencia
class EngineRule(object):
    """docstring for EngineRule"""

    def get_rules(self,a): # método encarrgado de extrair a regra do bd por meio de APU restfull e retorna um json da regra

        headers   = {'Authorization':'token %s' % "9517048ac92b9f9b5c7857e988580a66ba5d5061"}
        url       = 'http://localhost:8000/sensors/?format=json&uuID={0}'.format(a)
        r         = requests.get(url, headers=headers)
        getSensor = r.json()
        id_sensor =  getSensor[0]['id']
        url       = 'http://localhost:8000/rules/?format=json&sensor={0}'.format(id_sensor)
        r         = requests.get(url, headers=headers)
        rules     = r.json() # coleta os dados recebidos da APIrestfull

        return rules

    def get_parameters(self,obj_json): # pega os parametros enviados pelo tratador de evento e retorna um disct destes parametros

        # parameters = json.loads(obj_json)
        #print(parameters['valor'])
        return parameters

    def run_rules(self,a): # executa a regra

        parameters = self.get_parameters(a)
        obj_parameters= Parameters(parameters['id'],parameters['valor'],parameters['id_gateway']) #id_gateway futuramente será trabalhado
        rules = self.get_rules(parameters['id'])
        for i in range(0,len(rules),1): # percorre a lista que contem as regras
            #print(i)
            if(rules[i][jsonRule]):
                rule = json.loads(rules[i]['jsonRule']) # extrai as regras do json
                run_all(rule_list=rule,
                defined_variables=ConditionsRules(obj_parameters),
                defined_actions=ActionRules(obj_parameters),
                stop_on_first_trigger=True
                )

    def run_rules_t(self,a): # executa a regra

        obj_parameters= Parameters(10,30,10) #id_gateway futuramente será trabalhado
        rules = self.get_rules(a)
        # print(rules)
        for i in range(0,len(rules),1): # percorre a lista que contem as regras
            #print(i)
            if(rules[i]['status']):
                rule = json.loads(rules[i]['jsonRule']) # extrai as regras do json
                run_all(rule_list=rule,
                defined_variables=ConditionsRules(obj_parameters),
                defined_actions=ActionRules(obj_parameters),
                stop_on_first_trigger=True
                )
<commit_msg>fix bug engine rules<commit_after># -*- coding: utf-8 -*-
# Em teste
from business_rules import run_all
from business_rules.actions import BaseActions, rule_action
from business_rules.fields import FIELD_NUMERIC, FIELD_TEXT
from moduleOfRules.ActionRules import ActionRules
from moduleOfRules.ConditionsRules import ConditionsRules
from moduleOfRules.Parameters import Parameters
import requests
import json
import os
#metodo que recebe o json que contem regras de contigencia
class EngineRule(object):
    """docstring for EngineRule"""

    def get_rules(self,a): # método encarrgado de extrair a regra do bd por meio de APU restfull e retorna um json da regra

        headers   = {'Authorization':'token %s' % "9517048ac92b9f9b5c7857e988580a66ba5d5061"}
        url       = 'http://localhost:8000/sensors/?format=json&uuID={0}'.format(a)
        r         = requests.get(url, headers=headers)
        getSensor = r.json()
        id_sensor =  getSensor[0]['id']
        url       = 'http://localhost:8000/rules/?format=json&sensor={0}'.format(id_sensor)
        r         = requests.get(url, headers=headers)
        rules     = r.json() # coleta os dados recebidos da APIrestfull

        return rules

    def get_parameters(self,obj_json): # pega os parametros enviados pelo tratador de evento e retorna um disct destes parametros

        parameters = json.loads(obj_json)
        print(parameters['valor'])
        return parameters

    def run_rules(self,a): # executa a regra

        parameters = self.get_parameters(a)
        obj_parameters= Parameters(parameters['id'],parameters['valor'],parameters['id_gateway']) #id_gateway futuramente será trabalhado
        rules = self.get_rules(parameters['id'])
        for i in range(0,len(rules),1): # percorre a lista que contem as regras
            #print(i)
            if(rules[i][jsonRule]):
                rule = json.loads(rules[i]['jsonRule']) # extrai as regras do json
                run_all(rule_list=rule,
                defined_variables=ConditionsRules(obj_parameters),
                defined_actions=ActionRules(obj_parameters),
                stop_on_first_trigger=True
                )

    def run_rules_t(self,a): # executa a regra

        obj_parameters= Parameters(10,30,10) #id_gateway futuramente será trabalhado
        rules = self.get_rules(a)
        # print(rules)
        for i in range(0,len(rules),1): # percorre a lista que contem as regras
            #print(i)
            if(rules[i]['status']):
                rule = json.loads(rules[i]['jsonRule']) # extrai as regras do json
                run_all(rule_list=rule,
                defined_variables=ConditionsRules(obj_parameters),
                defined_actions=ActionRules(obj_parameters),
                stop_on_first_trigger=True
                )
<|endoftext|>


<commit_before>import pytest

import uuid
from io import BytesIO

from datetime import timedelta

from django.core import files
from django.utils import timezone

from evaluation.models import StagedFile
from evaluation.widgets.uploader import UploadedAjaxFileList, StagedAjaxFile

def create_uploaded_file(
        content: str,
        chunks=None,
        csrf="test_csrf",
        client_id="test_client_id",
        client_filename="test_client_filename_{uuid}",
        timeout=timedelta(minutes=1)) -> uuid.UUID:
    if chunks is None:
        chunks = [len(string)]

    new_uuid = uuid.uuid4()
    client_filename = client_filename.format(uuid=new_uuid)

    start = 0
    for chunk in chunks:
        staged_file = StagedFile(
            csrf=csrf,
            client_id=client_id,
            client_filename=client_filename,
            file_id=new_uuid,
            timeout=timezone.now() + timeout,
            start_byte=start,
            end_byte=chunk - 1,
            total_size=len(content),
        )

        string_file = BytesIO(content[start:chunk])
        django_file = files.File(string_file)
        staged_file.file.save(f"{client_filename}_{uuid.uuid4()}", django_file)

        staged_file.save()

        assert staged_file.file.size == chunk - start

        start = chunk + 1

    return new_uuid

@pytest.mark.django_db
def test_uploaded_file_assembly():
    file_content = b"HelloWorld" * 5
    uploaded_file_uuid = create_uploaded_file(
        file_content,
        [len(file_content)],
        client_filename="bla")

    testee = StagedAjaxFile(uploaded_file_uuid)

    assert testee.exists
    assert testee.is_complete
    assert testee.name == "bla"
    assert testee.size == len(file_content)

    with testee.open() as file:
        l = len("HelloWorld")
        assert file.read(l) == b"HelloWorld"
        assert file.read(l) == b"HelloWorld"

        assert file.seek(0) == file.tell()
        assert file.read() == file_content
<commit_msg>Added tests for checking that chunk cleaups work<commit_after>import pytest

import time
import uuid
from io import BytesIO

from datetime import timedelta

from django.core import files
from django.utils import timezone

from evaluation.models import StagedFile
from evaluation.widgets.uploader import UploadedAjaxFileList, StagedAjaxFile, \
    cleanup_stale_files

def create_uploaded_file(
        content: str,
        chunks=None,
        csrf="test_csrf",
        client_id="test_client_id",
        client_filename="test_client_filename_{uuid}",
        timeout=timedelta(minutes=1)) -> uuid.UUID:
    if chunks is None:
        chunks = [len(string)]

    new_uuid = uuid.uuid4()
    client_filename = client_filename.format(uuid=new_uuid)

    start = 0
    for chunk in chunks:
        staged_file = StagedFile(
            csrf=csrf,
            client_id=client_id,
            client_filename=client_filename,
            file_id=new_uuid,
            timeout=timezone.now() + timeout,
            start_byte=start,
            end_byte=chunk - 1,
            total_size=len(content),
        )

        string_file = BytesIO(content[start:chunk])
        django_file = files.File(string_file)
        staged_file.file.save(f"{client_filename}_{uuid.uuid4()}", django_file)

        staged_file.save()

        assert staged_file.file.size == chunk - start

        start = chunk + 1

    return new_uuid

@pytest.mark.django_db
def test_uploaded_file_assembly():
    file_content = b"HelloWorld" * 5
    uploaded_file_uuid = create_uploaded_file(
        file_content,
        [len(file_content)],
        client_filename="bla")

    testee = StagedAjaxFile(uploaded_file_uuid)

    assert testee.exists
    assert testee.is_complete
    assert testee.name == "bla"
    assert testee.size == len(file_content)

    with testee.open() as file:
        l = len("HelloWorld")
        assert file.read(l) == b"HelloWorld"
        assert file.read(l) == b"HelloWorld"

        assert file.seek(0) == file.tell()
        assert file.read() == file_content

@pytest.mark.django_db
def test_file_cleanup():
    file_content = b"HelloWorld" * 5
    uploaded_file_uuid = create_uploaded_file(
        file_content,
        [len(file_content)],
        client_filename="bla",
        timeout=timedelta(milliseconds=100))

    testee = StagedAjaxFile(uploaded_file_uuid)

    assert testee.exists
    chunks = StagedFile.objects.filter(file_id=testee.uuid).all()
    assert len(chunks) > 0

    # 200 ms > 100ms, time for cleaning!
    time.sleep(.2)
    cleanup_stale_files()

    assert not testee.exists

    chunks = StagedFile.objects.filter(file_id=testee.uuid).all()
    assert len(chunks) == 0
<|endoftext|>


<commit_before>import logging
import sys
import traceback
import xml.etree.ElementTree as ET
from django.core.management.base import BaseCommand

from ...models import Transcript, TranscriptMetadata, Topic, Source

logger = logging.getLogger('django')

errors = logging.getLogger('metadata_errors')

class Command(BaseCommand):
    help = 'Gets the AAPB PBCore information for a given transcript'

    def _topics(self, transcript, root):
        topic_tuples = [('science',
                         ['Science',
                          'Technology',
                          'Medicine',
                          'Nature',
                          'Environment',
                          'Energy',
                          'Animals',
                          'Psychology',
                          'Weather',
                          ]),
                        ('society',
                         ['Social Issues',
                          'War and Conflict',
                          'Women',
                          'Race and Ethnicity',
                          'Education',
                          'Employment',
                          'Law Enforcement and Crime',
                          'LGBTQ',
                          'Economics',
                          'Military Forces and Armaments',
                          'Agriculture',
                          'Health',
                          'Transportation',
                          'Parenting',
                          'Business',
                          ]),
                        ('politics',
                         ['Politics and Government',
                          'Global Affairs',
                          ]),
                        ('arts',
                         ['Theater',
                          'Performing Arts',
                          'Music',
                          'Architecture',
                          'Film and Television',
                          'Fine Arts',
                          'Dance',
                          'Humor',
                          ]),
                        ('news',
                         ['News',
                          'Journalism',
                          'Public Affairs',
                          'Local Communities',
                          ]),
                        ('humanities',
                         ['History',
                          'Biography',
                          'Philosophy',
                          'Religion',
                          'Geography',
                          'Literature',
                          ]),
                        ('home',
                         ['Food and Cooking',
                          'Home Improvement',
                          'Gardening',
                          'Consumer Affairs and Advocacy',
                          'Antiques and Collectibles',
                          'Crafts',
                          ])]
        topics = []
        for child in root.iter(
                '{http://www.pbcore.org/PBCore/PBCoreNamespace.html}pbcoreGenre'):
            if 'source' in child.attrib:
                if child.attrib['source'] == 'AAPB Topical Genre':
                    topics.append(child.text)

        generalized_topics = []
        for topic in set(topics):
            for t in topic_tuples:
                if topic in t[1]:
                    generalized_topics.append(t[0])

        for topic in set(generalized_topics):
            topic_object = Topic.objects.get(slug=topic)
            topic_object.transcripts.add(transcript)

    def _sources(self, transcript, root):
        source_candidates = []
        try:
            for child in root.iter(
                    '{http://www.pbcore.org/PBCore/PBCoreNamespace.html}pbcoreAnnotation'):
                if child.attrib['annotationType'] == 'organization':
                    logger.info('Found source {} for transcript: {}'.format(
                        child.text, transcript
                    ))
                    source_candidates.append(child.text)
        except:
            pass
        for candidate in set(source_candidates):
            source = Source.objects.get(pbcore_source=candidate)
            source.transcripts.add(transcript)

    def _description(self, root):
        try:
            for child in root.iter(
                    '{http://www.pbcore.org/PBCore/PBCoreNamespace.html}pbcoreDescription'):
                if child.attrib['descriptionType'] == 'Description':
                    return child.text
        except:
            return ''

    def _series_title(self, root):
        try:
            for child in root.iter(
                    '{http://www.pbcore.org/PBCore/PBCoreNamespace.html}pbcoreTitle'):
                if child.attrib['titleType'] == 'Series' or child.attrib[
                        'titleType'] == 'Program':
                    return child.text
        except:
            return ''

    def _broadcast_date(self, root):
        try:
            for child in root.iter(
                    '{http://www.pbcore.org/PBCore/PBCoreNamespace.html}pbcoreAssetDate'):
                return child.text
        except:
            return ''

    def handle(self, *args, **options):
        for transcript in Transcript.objects.filter(metadata_processed=False)[:5000]:
            try:
                tree = ET.ElementTree(ET.fromstringlist(transcript.aapb_xml))
                root = tree.getroot()
                self._topics(transcript, root)
                self._sources(transcript, root)
                metadata = {
                    'description': self._description(root),
                    'series': self._series_title(root),
                    'broadcast_date': self._broadcast_date(root),
                }
                tmd, created = TranscriptMetadata.objects.update_or_create(
                    transcript=transcript,
                    defaults=metadata,
                )
                tmd.save()
                transcript.metadata_processed = True
                transcript.save()
            except Exception as inst:
                errors.info('Problem transcript: {}'.format(transcript.name))
                errors.info(transcript.asset_name)
                errors.info('http://americanarchive.org/api/{}.xml'.format(transcript.asset_name))
                # errors.info(transcript.aapb_xml)
                errors.info(type(inst))
                errors.info(inst.args)
                errors.info(inst)
                tb = sys.exc_info()[2]
                errors.info(traceback.print_tb(tb))
                errors.info('\n')
                pass
<commit_msg>make it less error prone<commit_after>import logging
import sys
import traceback
import xml.etree.ElementTree as ET
from django.core.management.base import BaseCommand

from ...models import Transcript, TranscriptMetadata, Topic, Source

logger = logging.getLogger('django')

errors = logging.getLogger('metadata_errors')

class Command(BaseCommand):
    help = 'Gets the AAPB PBCore information for a given transcript'

    def _topics(self, transcript, root):
        topic_tuples = [('science',
                         ['Science',
                          'Technology',
                          'Medicine',
                          'Nature',
                          'Environment',
                          'Energy',
                          'Animals',
                          'Psychology',
                          'Weather',
                          ]),
                        ('society',
                         ['Social Issues',
                          'War and Conflict',
                          'Women',
                          'Race and Ethnicity',
                          'Education',
                          'Employment',
                          'Law Enforcement and Crime',
                          'LGBTQ',
                          'Economics',
                          'Military Forces and Armaments',
                          'Agriculture',
                          'Health',
                          'Transportation',
                          'Parenting',
                          'Business',
                          ]),
                        ('politics',
                         ['Politics and Government',
                          'Global Affairs',
                          ]),
                        ('arts',
                         ['Theater',
                          'Performing Arts',
                          'Music',
                          'Architecture',
                          'Film and Television',
                          'Fine Arts',
                          'Dance',
                          'Humor',
                          ]),
                        ('news',
                         ['News',
                          'Journalism',
                          'Public Affairs',
                          'Local Communities',
                          ]),
                        ('humanities',
                         ['History',
                          'Biography',
                          'Philosophy',
                          'Religion',
                          'Geography',
                          'Literature',
                          ]),
                        ('home',
                         ['Food and Cooking',
                          'Home Improvement',
                          'Gardening',
                          'Consumer Affairs and Advocacy',
                          'Antiques and Collectibles',
                          'Crafts',
                          ])]
        topics = []
        for child in root.iter(
                '{http://www.pbcore.org/PBCore/PBCoreNamespace.html}pbcoreGenre'):
            if 'source' in child.attrib:
                if child.attrib['source'] == 'AAPB Topical Genre':
                    topics.append(child.text)

        generalized_topics = []
        for topic in set(topics):
            for t in topic_tuples:
                if topic in t[1]:
                    generalized_topics.append(t[0])

        for topic in set(generalized_topics):
            topic_object = Topic.objects.get(slug=topic)
            topic_object.transcripts.add(transcript)

    def _sources(self, transcript, root):
        source_candidates = []
        try:
            for child in root.iter(
                    '{http://www.pbcore.org/PBCore/PBCoreNamespace.html}pbcoreAnnotation'):
                if child.attrib['annotationType'] == 'organization':
                    logger.info('Found source {} for transcript: {}'.format(
                        child.text, transcript
                    ))
                    source_candidates.append(child.text)
        except:
            pass
        for candidate in set(source_candidates):
            try:
                source = Source.objects.get(pbcore_source=candidate)
                source.transcripts.add(transcript)
            except:
                pass

    def _description(self, root):
        try:
            for child in root.iter(
                    '{http://www.pbcore.org/PBCore/PBCoreNamespace.html}pbcoreDescription'):
                if child.attrib['descriptionType'] == 'Description':
                    return child.text
        except:
            return ''

    def _series_title(self, root):
        try:
            for child in root.iter(
                    '{http://www.pbcore.org/PBCore/PBCoreNamespace.html}pbcoreTitle'):
                if child.attrib['titleType'] == 'Series' or child.attrib[
                        'titleType'] == 'Program':
                    return child.text
        except:
            return ''

    def _broadcast_date(self, root):
        try:
            for child in root.iter(
                    '{http://www.pbcore.org/PBCore/PBCoreNamespace.html}pbcoreAssetDate'):
                return child.text
        except:
            return ''

    def handle(self, *args, **options):
        for transcript in Transcript.objects.filter(metadata_processed=False)[:5000]:
            aapb_data = transcript.aapb_xml
            if aapb_data is not None:
                try:
                    tree = ET.ElementTree(ET.fromstringlist(aapb_data))
                    root = tree.getroot()
                    self._topics(transcript, root)
                    self._sources(transcript, root)
                    metadata = {
                        'description': self._description(root),
                        'series': self._series_title(root),
                        'broadcast_date': self._broadcast_date(root),
                    }
                    tmd, created = TranscriptMetadata.objects.update_or_create(
                        transcript=transcript,
                        defaults=metadata,
                    )
                    tmd.save()
                    transcript.metadata_processed = True
                    transcript.save()
                except Exception as inst:
                    errors.info('Problem transcript: {}'.format(transcript.name))
                    errors.info(transcript.asset_name)
                    errors.info('http://americanarchive.org/api/{}.xml'.format(transcript.asset_name))
                    errors.info(type(inst))
                    errors.info(inst.args)
                    errors.info(inst)
                    tb = sys.exc_info()[2]
                    errors.info(traceback.print_tb(tb))
                    errors.info('\n')
                    pass
<|endoftext|>


<commit_before>##
## For help on setting up your machine and configuring this TestScript go to
## http://docs.testdroid.com/appium/
##

import os
import time
import unittest
import subprocess
from appium import webdriver
from device_finder import DeviceFinder
from selenium.common.exceptions import WebDriverException
from testdroid_utils import TDUtils

class TestdroidAndroid(unittest.TestCase):
    def setUp(self):
        ##
        ## IMPORTANT: Set the following parameters.
        ## You can set the parameters outside the script with environment variables.
        ## If env var is not set the string after or is used.
        ##

        testdroid_url = os.environ.get('TESTDROID_URL') or "https://cloud.testdroid.com"
        appium_url = os.environ.get('TESTDROID_APPIUM_URL') or 'http://appium.testdroid.com/wd/hub'
        testdroid_apiKey = os.environ.get('TESTDROID_APIKEY') or ""
        testdroid_project = os.environ.get('TESTDROID_PROJECT') or 'Android hybrid sample project'
        testdroid_testrun = os.environ.get('TESTDROID_TESTRUN') or 'My testrun'
        testdroid_app = os.environ.get('TESTDROID_APP') or ""
        app_package = os.environ.get('TESTDROID_APP_PACKAGE') or 'com.testdroid.sample.android'
        app_activity = os.environ.get('TESTDROID_ACTIVITY') or '.MM_MainMenu'
        new_command_timeout = os.environ.get('TESTDROID_CMD_TIMEOUT') or '60'
        testdroid_test_timeout = os.environ.get('TESTDROID_TEST_TIMEOUT') or '600'

        self.screenshot_dir = os.environ.get('TESTDROID_SCREENSHOTS') or os.getcwd() + "/screenshots"
        self.screenshot_count = 1

        # Options to select device
        # 1) Set environment variable TESTDROID_DEVICE
        # 2) Set device name to this python script
        # 3) Do not set #1 and #2 and let DeviceFinder to find free device for you
        testdroid_device = os.environ.get('TESTDROID_DEVICE') or ""

        deviceFinder = DeviceFinder(url=testdroid_url)
        if testdroid_device == "":
            # Loop will not exit until free device is found
            while testdroid_device == "":
                testdroid_device = deviceFinder.available_android_device()

        self.utils = TDUtils(self.screenshot_dir)
        self.utils.log("Will save screenshots at: " + self.screenshot_dir)

        if "localhost" in appium_url:
            self.api_level = subprocess.check_output(["adb", "shell", "getprop ro.build.version.sdk"])
        else:
            self.api_level = deviceFinder.device_API_level(testdroid_device)

        self.utils.log("Device API level is %s" % self.api_level)
        self.utils.log("Starting Appium test using device '%s'" % testdroid_device)

        desired_capabilities_cloud = {}
        desired_capabilities_cloud['testdroid_apiKey'] = testdroid_apiKey
        if self.api_level > 16:
            desired_capabilities_cloud['testdroid_target'] = 'android'
            desired_capabilities_cloud['automationName'] = 'Appium'
        else:
            desired_capabilities_cloud['testdroid_target'] = 'selendroid'
            desired_capabilities_cloud['automationName'] = 'Selendroid'

        desired_capabilities_cloud['testdroid_apiKey'] = testdroid_apiKey
        desired_capabilities_cloud['testdroid_project'] = testdroid_project
        desired_capabilities_cloud['testdroid_testrun'] = testdroid_testrun
        desired_capabilities_cloud['testdroid_device'] = testdroid_device
        desired_capabilities_cloud['testdroid_app'] = testdroid_app
        desired_capabilities_cloud['appPackage'] = app_package
        desired_capabilities_cloud['appActivity'] = app_activity
        desired_capabilities_cloud['platformName'] = 'Android'
        desired_capabilities_cloud['deviceName'] = 'Android Phone'

        desired_capabilities_cloud['newCommandTimeout'] = new_command_timeout
        desired_capabilities_cloud['testdroid_testTimeout'] = testdroid_test_timeout

        # set up webdriver
        self.utils.log("WebDriver request initiated. Waiting for response, this typically takes 2-3 mins")
        self.driver = webdriver.Remote(appium_url, desired_capabilities_cloud)
        self.utils.update_driver(self.driver)
        self.utils.log("WebDriver response received")

    def tearDown(self):
        self.utils.log("Quitting")
        self.driver.quit()

    def testSample(self):
        self.utils.log("  Getting device screen size")
        self.utils.log("  " + str(self.driver.get_window_size()))

        isSelendroid = None
        if self.api_level < 17:
            isSelendroid = True

        self.utils.screenshot("app_launch")

        self.utils.log("Checking API level. This test works only on API 19 and above.")
        self.utils.log("API level: " + str(self.api_level))
        if self.api_level < 19:
            raise Exception("The chosen device has API level under 19. The Hybrid view will crash.")

        self.utils.log('Clicking button "hybrid app"')
        element = self.driver.find_element_by_id('com.testdroid.sample.android:id/mm_b_hybrid')
        element.click()
        self.utils.screenshot('hybrid_activity')

        url = "http://bitbar.github.io/testdroid-samples/"
        self.utils.log('Typing in the url ' + url)
        element = self.driver.find_element_by_id('com.testdroid.sample.android:id/hy_et_url')
        element.send_keys(url)
        self.utils.screenshot('url_typed')

        try:
            self.utils.log("Hiding keyboard")
            self.driver.hide_keyboard()
        except WebDriverException:
            pass  # pass exception, if keyboard isn't visible already
        self.utils.screenshot('keyboard_hidden')

        self.utils.log('Clicking Load url button')
        element = self.driver.find_element_by_id('com.testdroid.sample.android:id/hy_ib_loadUrl')
        element.click()
        self.utils.screenshot('webpage_loaded')

        context = "undefined"
        end_time = time.time() + 30
        while "undefined" in context and time.time() < end_time:
            contexts = self.driver.contexts
            context = str(contexts[-1])
            self.utils.log("Available contexts: {}, picking: {}".format(contexts, context))

        self.utils.log("Context will be " + context)
        self.driver.switch_to.context(context)
        self.utils.log("Context is " + self.driver.current_context)

        self.utils.log("Finding button with text 'Click for answer'")
        button = self.utils.wait_until_xpath_matches('//button[contains(., "Click for answer")]')

        self.utils.log("Clicking on button")
        button.click()
        self.utils.screenshot("answer")

        self.utils.log("Check answer text")
        self.driver.find_element_by_xpath('//p[@id="result_element" and contains(., "Testdroid")]')

        self.utils.log("Verify button changed color")
        style = str(button.get_attribute('style'))
        expected_style = "rgb(127, 255, 0"
        self.assertTrue(expected_style in style)

        self.driver.switch_to.context("NATIVE_APP")

        self.utils.log('Going back')
        self.driver.back()
        self.utils.screenshot('launch_screen')

def initialize():
    return TestdroidAndroid

if __name__ == "__main__":
    suite = unittest.TestLoader().loadTestsFromTestCase(TestdroidAndroid)
    unittest.TextTestRunner(verbosity=2).run(suite)
<commit_msg>Fix for Appium Hybrid test<commit_after>##
## For help on setting up your machine and configuring this TestScript go to
## http://docs.testdroid.com/appium/
##

import os
import time
import unittest
import subprocess
from appium import webdriver
from device_finder import DeviceFinder
from selenium.common.exceptions import WebDriverException
from testdroid_utils import TDUtils

class TestdroidAndroid(unittest.TestCase):
    def setUp(self):
        ##
        ## IMPORTANT: Set the following parameters.
        ## You can set the parameters outside the script with environment variables.
        ## If env var is not set the string after or is used.
        ##

        testdroid_url = os.environ.get('TESTDROID_URL') or "https://cloud.testdroid.com"
        appium_url = os.environ.get('TESTDROID_APPIUM_URL') or 'http://appium.testdroid.com/wd/hub'
        testdroid_apiKey = os.environ.get('TESTDROID_APIKEY') or ""
        testdroid_project = os.environ.get('TESTDROID_PROJECT') or 'Android hybrid sample project'
        testdroid_testrun = os.environ.get('TESTDROID_TESTRUN') or 'My testrun'
        testdroid_app = os.environ.get('TESTDROID_APP') or ""
        app_package = os.environ.get('TESTDROID_APP_PACKAGE') or 'com.testdroid.sample.android'
        app_activity = os.environ.get('TESTDROID_ACTIVITY') or '.MM_MainMenu'
        new_command_timeout = os.environ.get('TESTDROID_CMD_TIMEOUT') or '60'
        testdroid_test_timeout = os.environ.get('TESTDROID_TEST_TIMEOUT') or '600'

        self.screenshot_dir = os.environ.get('TESTDROID_SCREENSHOTS') or os.getcwd() + "/screenshots"
        self.screenshot_count = 1

        # Options to select device
        # 1) Set environment variable TESTDROID_DEVICE
        # 2) Set device name to this python script
        # 3) Do not set #1 and #2 and let DeviceFinder to find free device for you
        testdroid_device = os.environ.get('TESTDROID_DEVICE') or ""

        deviceFinder = DeviceFinder(url=testdroid_url)
        if testdroid_device == "":
            # Loop will not exit until free device is found
            while testdroid_device == "":
                testdroid_device = deviceFinder.available_android_device()

        self.utils = TDUtils(self.screenshot_dir)
        self.utils.log("Will save screenshots at: " + self.screenshot_dir)

        if "localhost" in appium_url:
            self.api_level = subprocess.check_output(["adb", "shell", "getprop ro.build.version.sdk"])
        else:
            self.api_level = deviceFinder.device_API_level(testdroid_device)

        self.utils.log("Device API level is %s" % self.api_level)
        self.utils.log("Starting Appium test using device '%s'" % testdroid_device)

        desired_capabilities_cloud = {}
        desired_capabilities_cloud['testdroid_apiKey'] = testdroid_apiKey
        if self.api_level > 16:
            desired_capabilities_cloud['testdroid_target'] = 'android'
            desired_capabilities_cloud['automationName'] = 'Appium'
        else:
            desired_capabilities_cloud['testdroid_target'] = 'selendroid'
            desired_capabilities_cloud['automationName'] = 'Selendroid'

        desired_capabilities_cloud['testdroid_apiKey'] = testdroid_apiKey
        desired_capabilities_cloud['testdroid_project'] = testdroid_project
        desired_capabilities_cloud['testdroid_testrun'] = testdroid_testrun
        desired_capabilities_cloud['testdroid_device'] = testdroid_device
        desired_capabilities_cloud['testdroid_app'] = testdroid_app
        desired_capabilities_cloud['appPackage'] = app_package
        desired_capabilities_cloud['appActivity'] = app_activity
        desired_capabilities_cloud['platformName'] = 'Android'
        desired_capabilities_cloud['deviceName'] = 'Android Phone'

        desired_capabilities_cloud['newCommandTimeout'] = new_command_timeout
        desired_capabilities_cloud['testdroid_testTimeout'] = testdroid_test_timeout

        # set up webdriver
        self.utils.log("WebDriver request initiated. Waiting for response, this typically takes 2-3 mins")
        self.driver = webdriver.Remote(appium_url, desired_capabilities_cloud)
        self.utils.update_driver(self.driver)
        self.utils.log("WebDriver response received")

    def tearDown(self):
        self.utils.log("Quitting")
        self.driver.quit()

    def testSample(self):
        self.utils.log("  Getting device screen size")
        self.utils.log("  " + str(self.driver.get_window_size()))

        isSelendroid = None
        if self.api_level < 17:
            isSelendroid = True

        self.utils.screenshot("app_launch")

        self.utils.log("Checking API level. This test works only on API 19 and above.")
        self.utils.log("API level: " + str(self.api_level))
        if self.api_level < 19:
            raise Exception("The chosen device has API level under 19. The Hybrid view will crash.")

        self.utils.log('Clicking button "hybrid app"')
        element = self.driver.find_element_by_id('com.testdroid.sample.android:id/mm_b_hybrid')
        element.click()
        self.utils.screenshot('hybrid_activity')

        url = "http://bitbar.github.io/testdroid-samples/"
        self.utils.log('Typing in the url ' + url)
        element = self.driver.find_element_by_id('com.testdroid.sample.android:id/hy_et_url')
        element.send_keys(url)
        self.utils.screenshot('url_typed')

        try:
            self.utils.log("Hiding keyboard")
            self.driver.hide_keyboard()
        except WebDriverException:
            pass  # pass exception, if keyboard isn't visible already
        self.utils.screenshot('keyboard_hidden')

        self.utils.log('Clicking Load url button')
        element = self.driver.find_element_by_id('com.testdroid.sample.android:id/hy_ib_loadUrl')
        element.click()
        self.utils.screenshot('webpage_loaded')

        context = "undefined"
        end_time = time.time() + 30
        while "undefined" in context and time.time() < end_time:
            contexts = self.driver.contexts
            context = str(contexts[-1])
            self.utils.log("Available contexts: {}, picking: {}".format(contexts, context))

        self.utils.log("Context will be " + context)
        self.driver.switch_to.context(context)
        self.utils.log("Context is " + self.driver.current_context)

        self.utils.log("Finding button with text 'Click for answer'")
        button = self.utils.wait_until_xpath_matches('//button[contains(., "Click for answer")]')

        self.utils.log("Clicking on button")
        button.click()
        self.utils.screenshot("answer")

        self.utils.log("Check answer text")
        self.driver.find_element_by_xpath('//p[@id="result_element" and contains(., "Bitbar")]')

        self.utils.log("Verify button changed color")
        style = str(button.get_attribute('style'))
        expected_style = "rgb(127, 255, 0"
        self.assertTrue(expected_style in style)

        self.driver.switch_to.context("NATIVE_APP")

        self.utils.log('Going back')
        self.driver.back()
        self.utils.screenshot('launch_screen')

def initialize():
    return TestdroidAndroid

if __name__ == "__main__":
    suite = unittest.TestLoader().loadTestsFromTestCase(TestdroidAndroid)
    unittest.TextTestRunner(verbosity=2).run(suite)
<|endoftext|>


<commit_before>import numpy as np
from scipy.spatial import cKDTree

# FIXME: dotriage should not be here
def triage(score, channel_index, triage_k, triage_percent, do_triage):
    """

    Note
    ----
    This object mutates score and clr_idx
    """

    K = triage_k + 1
    th = (1 - triage_percent)*100
    nmax = 10000

    n_data, n_features, n_channels = score.shape
    index_keep_bool = np.zeros(n_data,'bool')
    if n_data > nmax:
        index_keep = np.random.permutation(n_data)[:nmax]
        score = score[index_keep]
    else:
        index_keep = np.arange(n_data)

    if do_triage and (n_data > K):
        score_temp = score[:, :, channel_index]
        tree = cKDTree(score_temp)
        dist, ind = tree.query(score_temp, k=K)
        dist = np.sum(dist, 1)
        index_keep = index_keep[dist < np.percentile(dist, th)]

    index_keep_bool[index_keep] = 1

    return index_keep_bool<commit_msg>more pep8<commit_after>import numpy as np
from scipy.spatial import cKDTree

# FIXME: dotriage should not be here
# TODO: documentation
# TODO: comment code, it's not clear what it does
def triage(score, channel_index, triage_k, triage_percent, do_triage):
    """[?]

    Parameters
    ----------
    ?

    Note
    ----
    This object mutates score and clr_idx
    """

    K = triage_k + 1
    th = (1 - triage_percent)*100
    nmax = 10000

    n_data, n_features, n_channels = score.shape
    index_keep_bool = np.zeros(n_data, 'bool')
    if n_data > nmax:
        index_keep = np.random.permutation(n_data)[:nmax]
        score = score[index_keep]
    else:
        index_keep = np.arange(n_data)

    if do_triage and (n_data > K):
        score_temp = score[:, :, channel_index]
        tree = cKDTree(score_temp)
        dist, ind = tree.query(score_temp, k=K)
        dist = np.sum(dist, 1)
        index_keep = index_keep[dist < np.percentile(dist, th)]

    index_keep_bool[index_keep] = 1

    return index_keep_bool
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

"""

ZUGBRUECKE
Calling routines in Windows DLLs from Python scripts running on unixlike systems
https://github.com/pleiszenburg/zugbruecke

    src/zugbruecke/core/rpc.py: Customized RPC classes

    Required to run on platform / side: [UNIX, WINE]

    Copyright (C) 2017-2021 Sebastian M. Ernst <ernst@pleiszenburg.de>

<LICENSE_BLOCK>
The contents of this file are subject to the GNU Lesser General Public License
Version 2.1 ("LGPL" or "License"). You may not use this file except in
compliance with the License. You may obtain a copy of the License at
https://www.gnu.org/licenses/old-licenses/lgpl-2.1.txt
https://github.com/pleiszenburg/zugbruecke/blob/master/LICENSE

Software distributed under the License is distributed on an "AS IS" basis,
WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License for the
specific language governing rights and limitations under the License.
</LICENSE_BLOCK>

"""

# +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
# IMPORT
# +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++

from multiprocessing.connection import Client, Listener, _ConnectionBase
from threading import Thread
import time
import traceback
from typing import Any, Callable, Tuple, Union

from .abc import LogABC, RpcClientABC, RpcServerABC
from .typeguard import typechecked

# +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
# CLASSES
# +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++

@typechecked
class RpcClient(RpcClientABC):
    """
    RPC client
    """

    def __init__(self, socket_path: Tuple[str, int], authkey: str):

        self._client = Client(socket_path, authkey=authkey.encode("utf-8"))
        self._cache = {}

    def __getattr__(self, name: str) -> Callable:

        try:
            return self._cache[name]
        except KeyError:
            pass

        def call_rpc_server(*args: Any, **kwargs: Any) -> Any:

            self._client.send((name, args, kwargs))
            result = self._client.recv()

            if isinstance(result, Exception):
                # TODO print traceback to stderr?
                raise result

            return result

        self._cache[name] = call_rpc_server
        return call_rpc_server

    @classmethod
    def from_safe_connect(cls,
        socket_path: Tuple[str, int],
        authkey: str,
        timeout_after_seconds: Union[int, float] = 30,
        wait_for_seconds: Union[int, float] = 0.01,
    ) -> RpcClientABC:

        started_waiting_at = time.time()

        while True:

            try:
                client = cls(socket_path, authkey)
                if client.get_rpc_status():
                    return client
            except:
                pass

            if time.time() >= (started_waiting_at + timeout_after_seconds):
                break

            time.sleep(wait_for_seconds)

        raise TimeoutError("rpc client failed to start")

@typechecked
class RpcServer(RpcServerABC):
    """
    RPC client
    """

    def __init__(self,
        socket_path: Tuple[str, int],
        authkey: str,
        log: Union[LogABC, None] = None,
        terminate_function: Union[Callable, None] = None,
    ):

        self._up = True
        self._log = log

        if self._log is not None:
            self._log.out("[rpc-server] STARTING ...")
            self._log.out("[rpc-server] Log attached.")

        self._socket_path = socket_path
        self._authkey = authkey.encode("utf-8")
        self._terminate_function = terminate_function

        self._server = None
        self._t = None

        self._functions = {}
        self.register_function(self.get_rpc_status)

        if self._log is not None:
            self._log.out("[rpc-server] STARTED.")

    def get_rpc_status(self) -> bool:
        """
        Called by RPC client
        """

        return self._up

    def register_function(self, function_pointer: Callable, public_name: Union[str, None] = None):

        self._functions[function_pointer.__name__ if public_name is None else public_name] = function_pointer

    def terminate(self):

        if not self._up:
            return

        self._up = False

        if self._log is not None:
            self._log.out("[rpc-server] TERMINATING ...")

        if self._terminate_function is not None:
            self._terminate_function()

        if self._log is not None:
            self._log.out("[rpc-server] TERMINATED.")

        self._server.close()

    def serve_forever(self):

        self._server = Listener(self._socket_path, authkey=self._authkey)

        while self._up:

            try:
                connection = self._server.accept()
                t = Thread(target=self._handle_connection, args=(connection,))
                t.daemon = True
                t.start()
            except OSError:
                if self._log is not None:
                    self._log.out("[rpc-server] OSError: Socket likely closed.")
            except:
                traceback.print_exc()

    def server_forever_in_thread(self, daemon: bool = True):

        if self._t is not None:
            return

        self._t = Thread(target=self.serve_forever)
        self._t.daemon = daemon
        self._t.start()

    def _handle_connection(self, connection: _ConnectionBase):

        try:
            while True:
                function_name, args, kwargs = connection.recv()
                try:
                    r = self._functions[function_name](*args, **kwargs)
                    connection.send(r)
                except Exception as e:
                    connection.send(e)
        except EOFError:
            pass
<commit_msg>black<commit_after># -*- coding: utf-8 -*-

"""

ZUGBRUECKE
Calling routines in Windows DLLs from Python scripts running on unixlike systems
https://github.com/pleiszenburg/zugbruecke

    src/zugbruecke/core/rpc.py: Customized RPC classes

    Required to run on platform / side: [UNIX, WINE]

    Copyright (C) 2017-2021 Sebastian M. Ernst <ernst@pleiszenburg.de>

<LICENSE_BLOCK>
The contents of this file are subject to the GNU Lesser General Public License
Version 2.1 ("LGPL" or "License"). You may not use this file except in
compliance with the License. You may obtain a copy of the License at
https://www.gnu.org/licenses/old-licenses/lgpl-2.1.txt
https://github.com/pleiszenburg/zugbruecke/blob/master/LICENSE

Software distributed under the License is distributed on an "AS IS" basis,
WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License for the
specific language governing rights and limitations under the License.
</LICENSE_BLOCK>

"""

# +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
# IMPORT
# +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++

from multiprocessing.connection import Client, Listener, _ConnectionBase
from threading import Thread
import time
import traceback
from typing import Any, Callable, Tuple, Union

from .abc import LogABC, RpcClientABC, RpcServerABC
from .typeguard import typechecked

# +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
# CLASSES
# +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++

@typechecked
class RpcClient(RpcClientABC):
    """
    RPC client
    """

    def __init__(self, socket_path: Tuple[str, int], authkey: str):

        self._client = Client(socket_path, authkey=authkey.encode("utf-8"))
        self._cache = {}

    def __getattr__(self, name: str) -> Callable:

        try:
            return self._cache[name]
        except KeyError:
            pass

        def call_rpc_server(*args: Any, **kwargs: Any) -> Any:

            self._client.send((name, args, kwargs))
            result = self._client.recv()

            if isinstance(result, Exception):
                # TODO print traceback to stderr?
                raise result

            return result

        self._cache[name] = call_rpc_server
        return call_rpc_server

    @classmethod
    def from_safe_connect(
        cls,
        socket_path: Tuple[str, int],
        authkey: str,
        timeout_after_seconds: Union[int, float] = 30,
        wait_for_seconds: Union[int, float] = 0.01,
    ) -> RpcClientABC:

        started_waiting_at = time.time()

        while True:

            try:
                client = cls(socket_path, authkey)
                if client.get_rpc_status():
                    return client
            except:
                pass

            if time.time() >= (started_waiting_at + timeout_after_seconds):
                break

            time.sleep(wait_for_seconds)

        raise TimeoutError("rpc client failed to start")

@typechecked
class RpcServer(RpcServerABC):
    """
    RPC client
    """

    def __init__(
        self,
        socket_path: Tuple[str, int],
        authkey: str,
        log: Union[LogABC, None] = None,
        terminate_function: Union[Callable, None] = None,
    ):

        self._up = True
        self._log = log

        if self._log is not None:
            self._log.out("[rpc-server] STARTING ...")
            self._log.out("[rpc-server] Log attached.")

        self._socket_path = socket_path
        self._authkey = authkey.encode("utf-8")
        self._terminate_function = terminate_function

        self._server = None
        self._t = None

        self._functions = {}
        self.register_function(self.get_rpc_status)

        if self._log is not None:
            self._log.out("[rpc-server] STARTED.")

    def get_rpc_status(self) -> bool:
        """
        Called by RPC client
        """

        return self._up

    def register_function(
        self, function_pointer: Callable, public_name: Union[str, None] = None
    ):

        self._functions[
            function_pointer.__name__ if public_name is None else public_name
        ] = function_pointer

    def terminate(self):

        if not self._up:
            return

        self._up = False

        if self._log is not None:
            self._log.out("[rpc-server] TERMINATING ...")

        if self._terminate_function is not None:
            self._terminate_function()

        if self._log is not None:
            self._log.out("[rpc-server] TERMINATED.")

        self._server.close()

    def serve_forever(self):

        self._server = Listener(self._socket_path, authkey=self._authkey)

        while self._up:

            try:
                connection = self._server.accept()
                t = Thread(target=self._handle_connection, args=(connection,))
                t.daemon = True
                t.start()
            except OSError:
                if self._log is not None:
                    self._log.out("[rpc-server] OSError: Socket likely closed.")
            except:
                traceback.print_exc()

    def server_forever_in_thread(self, daemon: bool = True):

        if self._t is not None:
            return

        self._t = Thread(target=self.serve_forever)
        self._t.daemon = daemon
        self._t.start()

    def _handle_connection(self, connection: _ConnectionBase):

        try:
            while True:
                function_name, args, kwargs = connection.recv()
                try:
                    r = self._functions[function_name](*args, **kwargs)
                    connection.send(r)
                except Exception as e:
                    connection.send(e)
        except EOFError:
            pass
<|endoftext|>


<commit_before>import os

from unittest.mock import patch
from unittest import TestCase

from zulint.custom_rules import RuleList
from linter_lib.custom_check import python_rules, non_py_rules

ROOT_DIR = os.path.abspath(os.path.join(__file__, '..', '..', '..'))
CHECK_MESSAGE = "Fix the corresponding rule in `tools/linter_lib/custom_check.py`."

class TestRuleList(TestCase):

    def setUp(self) -> None:
        self.all_rules = python_rules.rules
        for rule in non_py_rules:
            self.all_rules.extend(rule.rules)

    def test_paths_in_rules(self) -> None:
        """Verifies that the paths mentioned in linter rules actually exist"""
        for rule in self.all_rules:
            for path in rule.get('exclude', {}):
                abs_path = os.path.abspath(os.path.join(ROOT_DIR, path))
                self.assertTrue(os.path.exists(abs_path),
                                "'{}' is neither an existing file, nor a directory. {}".format(path, CHECK_MESSAGE))

            for line_tuple in rule.get('exclude_line', {}):
                path = line_tuple[0]
                abs_path = os.path.abspath(os.path.join(ROOT_DIR, path))
                self.assertTrue(os.path.isfile(abs_path),
                                "The file '{}' doesn't exist. {}".format(path, CHECK_MESSAGE))

            for path in rule.get('include_only', {}):
                if not os.path.splitext(path)[1]:
                    self.assertTrue(path.endswith('/'),
                                    "The path '{}' should end with '/'. {}".format(path, CHECK_MESSAGE))

    def test_rule_patterns(self) -> None:
        """Verifies that the search regex specified in a custom rule actually matches
           the expectation and doesn't throw false positives."""
        for rule in self.all_rules:
            pattern = rule['pattern']
            for line in rule.get('good_lines', []):
                # create=True is superfluous when mocking built-ins in Python >= 3.5
                with patch('builtins.open', return_value=iter((line+'\n\n').splitlines()), create=True, autospec=True):
                    self.assertFalse(RuleList([], [rule]).custom_check_file('foo.bar', 'baz', ''),
                                     "The pattern '{}' matched the line '{}' while it shouldn't.".format(pattern, line))

            for line in rule.get('bad_lines', []):
                # create=True is superfluous when mocking built-ins in Python >= 3.5
                with patch('builtins.open',
                           return_value=iter((line+'\n\n').splitlines()), create=True, autospec=True), patch('builtins.print'):
                    filename = list(rule.get('include_only', {'foo.bar'}))[0]
                    self.assertTrue(RuleList([], [rule]).custom_check_file(filename, 'baz', ''),
                                    "The pattern '{}' didn't match the line '{}' while it should.".format(pattern, line))
<commit_msg>test_zulint_custom_rules: Fix open mock.<commit_after>from io import StringIO
import os

from unittest.mock import patch
from unittest import TestCase

from zulint.custom_rules import RuleList
from linter_lib.custom_check import python_rules, non_py_rules

ROOT_DIR = os.path.abspath(os.path.join(__file__, '..', '..', '..'))
CHECK_MESSAGE = "Fix the corresponding rule in `tools/linter_lib/custom_check.py`."

class TestRuleList(TestCase):

    def setUp(self) -> None:
        self.all_rules = python_rules.rules
        for rule in non_py_rules:
            self.all_rules.extend(rule.rules)

    def test_paths_in_rules(self) -> None:
        """Verifies that the paths mentioned in linter rules actually exist"""
        for rule in self.all_rules:
            for path in rule.get('exclude', {}):
                abs_path = os.path.abspath(os.path.join(ROOT_DIR, path))
                self.assertTrue(os.path.exists(abs_path),
                                "'{}' is neither an existing file, nor a directory. {}".format(path, CHECK_MESSAGE))

            for line_tuple in rule.get('exclude_line', {}):
                path = line_tuple[0]
                abs_path = os.path.abspath(os.path.join(ROOT_DIR, path))
                self.assertTrue(os.path.isfile(abs_path),
                                "The file '{}' doesn't exist. {}".format(path, CHECK_MESSAGE))

            for path in rule.get('include_only', {}):
                if not os.path.splitext(path)[1]:
                    self.assertTrue(path.endswith('/'),
                                    "The path '{}' should end with '/'. {}".format(path, CHECK_MESSAGE))

    def test_rule_patterns(self) -> None:
        """Verifies that the search regex specified in a custom rule actually matches
           the expectation and doesn't throw false positives."""
        for rule in self.all_rules:
            pattern = rule['pattern']
            for line in rule.get('good_lines', []):
                # create=True is superfluous when mocking built-ins in Python >= 3.5
                with patch('builtins.open', return_value=StringIO(line + '\n\n'), create=True, autospec=True):
                    self.assertFalse(RuleList([], [rule]).custom_check_file('foo.bar', 'baz', ''),
                                     "The pattern '{}' matched the line '{}' while it shouldn't.".format(pattern, line))

            for line in rule.get('bad_lines', []):
                # create=True is superfluous when mocking built-ins in Python >= 3.5
                with patch('builtins.open',
                           return_value=StringIO(line + '\n\n'), create=True, autospec=True), patch('builtins.print'):
                    filename = list(rule.get('include_only', {'foo.bar'}))[0]
                    self.assertTrue(RuleList([], [rule]).custom_check_file(filename, 'baz', ''),
                                    "The pattern '{}' didn't match the line '{}' while it should.".format(pattern, line))
<|endoftext|>


<commit_before>from django.http import HttpResponse

from .utils import check_input

from ifind.search import Query
from ifind.search import EngineFactory

def search(request):
    """
    Accepts GET request containing query terms,
    searches and returns results as JSON.

    """
    # invalid HTTP method
    if request.method != 'GET':
        return HttpResponse(status=405)

    # checks input for validity
    query_terms = check_input(request.GET.get('q', ""))

    # bad request
    if not query_terms:
        return HttpResponse(status=400)

    # execute query
    engine = EngineFactory('govuk')
    query = Query(query_terms, top=50)
    response = engine.search(query)

    return HttpResponse(response.to_json(), content_type='application/json')

<commit_msg>created session test view<commit_after>from django.http import HttpResponse

from .utils import check_input
from .utils import get_client_ip

from ifind.search import Query
from ifind.search import EngineFactory

def search(request):
    """
    Accepts GET request containing query terms,
    searches and returns results as JSON.

    """
    # invalid HTTP method
    if request.method != 'GET':
        return HttpResponse(status=405)

    # checks input for validity
    query_terms = check_input(request.GET.get('q', ""))

    # bad request
    if not query_terms:
        return HttpResponse(status=400)

    # execute query
    engine = EngineFactory('govuk')
    query = Query(query_terms, top=50)
    response = engine.search(query)

    ##########

    print '********************* -- ' + get_client_ip(request)

    ##########

    return HttpResponse(response.to_json(), content_type='application/json')

def session(request):

    if "animal_type" in request.session:
        return HttpResponse(request.session['animal_type'])

    response = HttpResponse("ducks")
    request.session["animal_type"] = "goose"

    return response<|endoftext|>


<commit_before>from .model_not_found_exception import ModelNotFoundException

class Importer:
    def __init__(self, collection):
        self.collection = collection
        self.model = False
        self.model_fields = []

    def import_note(self, import_data):
        self.__load_model(import_data['model'])
        note = self.__find_or_create_note(import_data['id'])
        self.__populate_note_fields(
            note, import_data['fields'], import_data['id'])

        # Set note deck
        note.model()['did'] = self.collection.decks.id(import_data['deck'])

        # Save changes
        is_new = not hasattr(note, 'mod')
        note.flush()

        # If note is new, add to collection
        if is_new:
            self.collection.addNote(note)

        return True

    def __load_model(self, model_name):
        # Get model
        model = self.collection.models.byName(model_name)
        if model is None:
            raise ModelNotFoundException(model_name)

        # Set current model
        self.collection.models.setCurrent(model)
        self.model = model
        self.model_fields = self.collection.models.fieldNames(self.model)

    def __populate_note_fields(self, note, fields, node_id):
        # Set id field to Freeplane node id
        id_field = self.__get_model_id_field()
        if id_field is not None:
            note[id_field] = node_id

        # Populate remaining fields
        for field in self.model_fields:
            if field in fields:
                note[field] = fields[field]

    def __get_model_id_field(self):
        if len(self.model_fields) > 0 and self.model_fields[0].lower() == 'id':
            return self.model_fields[0]
        else:
            return None

    def __find_or_create_note(self, node_id):
        existing_id = self.collection.db.scalar(
            'select id from notes where flds LIKE ? AND mid = ?', str(node_id) + "\x1f%", self.model['id'])
        if existing_id is not None:
            note = self.collection.getNote(existing_id)
            if note is not None:
                return note

        # Existing note wasn't found
        return self.collection.newNote()
<commit_msg>Capitalise database keywords<commit_after>from .model_not_found_exception import ModelNotFoundException

class Importer:
    def __init__(self, collection):
        self.collection = collection
        self.model = False
        self.model_fields = []

    def import_note(self, import_data):
        self.__load_model(import_data['model'])
        note = self.__find_or_create_note(import_data['id'])
        self.__populate_note_fields(
            note, import_data['fields'], import_data['id'])

        # Set note deck
        note.model()['did'] = self.collection.decks.id(import_data['deck'])

        # Save changes
        is_new = not hasattr(note, 'mod')
        note.flush()

        # If note is new, add to collection
        if is_new:
            self.collection.addNote(note)

        return True

    def __load_model(self, model_name):
        # Get model
        model = self.collection.models.byName(model_name)
        if model is None:
            raise ModelNotFoundException(model_name)

        # Set current model
        self.collection.models.setCurrent(model)
        self.model = model
        self.model_fields = self.collection.models.fieldNames(self.model)

    def __populate_note_fields(self, note, fields, node_id):
        # Set id field to Freeplane node id
        id_field = self.__get_model_id_field()
        if id_field is not None:
            note[id_field] = node_id

        # Populate remaining fields
        for field in self.model_fields:
            if field in fields:
                note[field] = fields[field]

    def __get_model_id_field(self):
        if len(self.model_fields) > 0 and self.model_fields[0].lower() == 'id':
            return self.model_fields[0]
        else:
            return None

    def __find_or_create_note(self, node_id):
        existing_id = self.collection.db.scalar(
            'SELECT id FROM notes WHERE flds LIKE ? AND mid = ?',
            str(node_id) + "\x1f%",
            self.model['id']
        )

        if existing_id is not None:
            note = self.collection.getNote(existing_id)
            if note is not None:
                return note

        # Existing note wasn't found
        return self.collection.newNote()
<|endoftext|>