<commit_before>from __future__ import division

import string
import random
import numpy as np
from scipy.odr import *

from summer import sum_function

LIBRARY = [i/6.0 for i in range(1,151)]+[0.35+i/2000 for i in range(1,100)]+[0.05+0.001*i for i in range(1,100)]

class Sample(object):
    def __init__(self,v):
        self.v = v

class GeneticHelloWorld(object):
    def __init__(self,
               num_samples = 80, # Have 60 chromos in the sample group
               num_selected = 20, # Have 10 chromos in the selected group
               mutation_factor = 2, # Randomly mutate 1/n of the chromosomes
               num_powers = 4, # How many powers in the fit
               Ref = 'REFPROP-R134a',
               value = 'rhoV',
               addTr = False
                ):
        self.num_samples = num_samples
        self.num_selected = num_selected
        self.mutation_factor = mutation_factor
        self.num_powers = num_powers
        self.addTr = addTr
        self.value = value
        self.Ref = Ref

        #THermodynamics
        from CoolProp.CoolProp import Props
        self.Tc = Props(Ref,'Tcrit')
        self.pc = Props(Ref,'pcrit')
        self.rhoc = Props(Ref,'rhocrit')
        self.Tmin = Props(Ref,'Tmin')

        self.T = np.linspace(self.Tmin+1e-6, self.Tc-0.000001,200)
        self.p = [Props('P','T',T,'Q',0,Ref) for T in self.T]
        self.rhoL = [Props('D','T',T,'Q',0,Ref) for T in self.T]
        self.rhoV = [Props('D','T',T,'Q',1,Ref) for T in self.T]
        self.logppc = (np.log(self.p)-np.log(self.pc))
        self.rhoLrhoc = np.array(self.rhoL)/self.rhoc
        self.rhoVrhoc = np.array(self.rhoV)/self.rhoc
        self.logrhoLrhoc = np.log(self.rhoL)-np.log(self.rhoc)
        self.logrhoVrhoc = np.log(self.rhoV)-np.log(self.rhoc)

    def generate_random_chromosomes(self,):
        '''
        Create a list of random chromosomes to seed our alogrithm.
        '''
        chromos = []
        while len(chromos) < self.num_samples:
            chromos.append(Sample(sorted(random.sample(LIBRARY,self.num_powers))))
        return chromos

    def fitness(self, chromo):
        '''
        Fitness of a chromo is the sum of the squares of the error of the correlation
        '''
        x = 1.0-self.T/self.Tc

        if self.value == 'p':
            LHS = self.logppc
        elif self.value == 'rhoL':
            LHS = self.logrhoLrhoc
        elif self.value == 'rhoV':
            LHS = self.logrhoVrhoc
        elif self.value == 'rhoLnoexp':
            LHS = self.rhoLrhoc-1

        if self.addTr:
            LHS *= self.T/self.Tc

        output = np.zeros_like(self.T)

        def f_RHS(B, x):
            sum_function(B,x,np.array(chromo.v),output)
            return output

        linear = Model(f_RHS)
        mydata = Data(x, LHS)
        myodr = ODR(mydata, linear, beta0=[0.0]*self.num_powers)
        myoutput = myodr.run()

        chromo.beta = myoutput.beta

        if self.addTr:
            RHS = f_RHS(myoutput.beta,x)*self.Tc/self.T
        else:
            RHS = f_RHS(myoutput.beta,x)

        if self.value == 'p':
            fit_value = np.exp(RHS)*self.pc
            EOS_value = self.p
        elif self.value in ['rhoL','rhoV']:
            fit_value = np.exp(RHS)*self.rhoc
            if self.value == 'rhoL':
                EOS_value = self.rhoL
            elif self.value == 'rhoV':
                EOS_value = self.rhoV
        else:
            fit_value = self.rhoc*(1+RHS)
            EOS_value = self.rhoL

        max_abserror = np.max(np.abs((fit_value/EOS_value)-1)*100)

#        import matplotlib.pyplot as plt
#        plt.plot(x,fit_value,x,EOS_value)
#        plt.show()

        chromo.fitness = max_abserror
        chromo.sum_square = myoutput.sum_square
        chromo.max_abserror = max_abserror

        #print self.Ref, 'rhoL SSE =', myoutput.sum_square, myoutput.beta, max_abserror,'%'
        print '.',

        return chromo.fitness

    def tourny_select_chromo(self, samples):
        '''
        Randomly select two chromosomes from the samples, then return the one
        with the best fitness.
        '''
        a = random.choice(samples)
        b = random.choice(samples)
        if a.fitness < b.fitness:
            return a
        else:
            return b

    def breed(self, a, b):
        '''
        Breed two chromosomes by splicing them in a random spot and combining
        them together to form two new chromos.
        '''
        splice_pos = random.randrange(len(a.v))
        new_a = a.v[:splice_pos] + b.v[splice_pos:]
        new_b = b.v[:splice_pos] + a.v[splice_pos:]
        return Sample(sorted(new_a)), Sample(sorted(new_b))

    def mutate(self, chromo):
        '''
        Mutate a chromosome by changing one of the parameters, but only if it improves the fitness
        '''
        v = chromo.v
        if hasattr(chromo,'fitness'):
            old_fitness = chromo.fitness
        else:
            old_fitness = self.fitness(chromo)

        pos = random.randrange(len(chromo.v))
        chromo.v[pos] = random.choice(LIBRARY)
        new_fitness = self.fitness(chromo)
        if new_fitness < old_fitness:
            return chromo
        else:
            return Sample(sorted(v))

    def run(self):
        # Create a random sample of chromos
        samples = self.generate_random_chromosomes()

        # Calculate the fitness for the initial chromosomes
        for chromo in samples:
            self.fitness(chromo)
        print '#'

        decorated = [(sample.fitness,sample) for sample in samples]
        decorated.sort()
        samples = [s for sv,s in decorated]

        # Main loop: each generation select a subset of the sample and breed from
        # them.
        generation = -1
        while generation < 0 or samples[0].fitness > 0.1 or generation < 3 and generation < 15:
            generation += 1

            # Generate the selected group from sample- take the top 10% of samples
            # and tourny select to generate the rest of selected.
            ten_percent = int(len(samples)*.1)
            selected = samples[:ten_percent]
            while len(selected) < self.num_selected:
                selected.append(self.tourny_select_chromo(samples))

            # Generate the solution group by breeding random chromos from selected
            solution = []
            while len(solution) < self.num_samples:
                solution.extend(self.breed(random.choice(selected),
                                         random.choice(selected)))

            # Apply a mutation to a subset of the solution set
            mutate_indices = random.sample(range(len(solution)), len(solution)//self.mutation_factor)
            for i in mutate_indices:
                solution[i] = self.mutate(solution[i])

            for chromo in solution:
                self.fitness(chromo)
            print '#'

            decorated = [(sample.fitness,sample) for sample in solution]
            decorated.sort()
            samples = [s for sv,s in decorated]

            print '------------------  Top 10 values  ---------------'
            for sample in samples[0:10]:
                print sample.v, sample.fitness, sample.max_abserror

            print samples[0].v, list(samples[0].beta)

            # Print useful stats about this generation
            (min, median, max) =  [samples[0].fitness, samples[len(samples)//2].fitness, samples[-1].fitness]
            print("{0} best value: {1}. fitness: best {2}, median {3}, worst {4}".format(generation, samples[0].v, min, median, max))

        return samples[0]

def main():
    ghw = GeneticHelloWorld()
    r = ghw.run()

    print r.v
    print list(r.beta)

if __name__ == "__main__":
  main() <commit_msg>Fixed genetic solver<commit_after>from __future__ import division

import string
import random
import numpy as np
from scipy.odr import *

import matplotlib.pyplot as plt

from summer import sum_function

LIBRARY = [i/6.0 for i in range(1,151)]+[0.35+i/2000 for i in range(1,100)]+[0.05+0.001*i for i in range(1,100)]+[i+0.5 for i in range(10)]
#LIBRARY = [i+0.5 for i in range(10)]+[i+0.0 for i in range(10)]

class Sample(object):
    def __init__(self,v):
        self.v = v

class GeneticAncillaryFitter(object):
    def __init__(self,
               num_samples = 100, # Have this many chromos in the sample group
               num_selected = 30, # Have this many chromos in the selected group
               mutation_factor = 2, # Randomly mutate 1/n of the chromosomes
               num_powers = 6, # How many powers in the fit
               Ref = 'o-Xylene',
               value = 'rhoV',
               addTr = True
                ):
        self.num_samples = num_samples
        self.num_selected = num_selected
        self.mutation_factor = mutation_factor
        self.num_powers = num_powers
        self.addTr = addTr
        self.value = value
        self.Ref = Ref

        #THermodynamics
        from CoolProp.CoolProp import Props
        self.Tc = Props(Ref,'Tcrit')
        self.pc = Props(Ref,'pcrit')
        self.rhoc = Props(Ref,'rhocrit')
        self.Tmin = Props(Ref,'Tmin')

        self.T = np.linspace(self.Tmin+1e-6, self.Tc-0.000001,200)
        self.p = [Props('P','T',T,'Q',0,Ref) for T in self.T]
        self.rhoL = [Props('D','T',T,'Q',0,Ref) for T in self.T]
        self.rhoV = [Props('D','T',T,'Q',1,Ref) for T in self.T]
        self.logppc = (np.log(self.p)-np.log(self.pc))
        self.rhoLrhoc = np.array(self.rhoL)/self.rhoc
        self.rhoVrhoc = np.array(self.rhoV)/self.rhoc
        self.logrhoLrhoc = np.log(self.rhoL)-np.log(self.rhoc)
        self.logrhoVrhoc = np.log(self.rhoV)-np.log(self.rhoc)

        self.x = 1.0-self.T/self.Tc

        if self.value == 'p':
            self.LHS = self.logppc.copy()
        elif self.value == 'rhoL':
            self.LHS = self.logrhoLrhoc.copy()
        elif self.value == 'rhoV':
            self.LHS = self.logrhoVrhoc.copy()
        elif self.value == 'rhoLnoexp':
            self.LHS = (self.rhoLrhoc-1).copy()

        if self.addTr:
            self.LHS *= self.T/self.Tc

    def generate_random_chromosomes(self,):
        '''
        Create a list of random chromosomes to seed our alogrithm.
        '''
        chromos = []
        while len(chromos) < self.num_samples:
            chromos.append(Sample(sorted(random.sample(LIBRARY,self.num_powers))))
        return chromos

    def fitness(self, chromo):
        '''
        Fitness of a chromo is the sum of the squares of the error of the correlation
        '''

        output = np.zeros_like(self.T)

        def f_RHS(B,x):
            sum_function(B,x,np.array(chromo.v),output)
            return output

        linear = Model(f_RHS)
        mydata = Data(self.x, self.LHS)
        myodr = ODR(mydata, linear, beta0=[0.0]*self.num_powers)#, maxit = 100, sstol = 1e-10, partol = 1e-10)
        myoutput = myodr.run()

        chromo.beta = myoutput.beta

        if self.addTr:
            RHS = f_RHS(myoutput.beta, self.x)*self.Tc/self.T
        else:
            RHS = f_RHS(myoutput.beta, self.x)

        if self.value == 'p':
            fit_value = np.exp(RHS)*self.pc
            EOS_value = self.p
        elif self.value in ['rhoL','rhoV']:
            fit_value = np.exp(RHS)*self.rhoc
            if self.value == 'rhoL':
                EOS_value = self.rhoL
            elif self.value == 'rhoV':
                EOS_value = self.rhoV
        else:
            fit_value = self.rhoc*(1+RHS)
            EOS_value = self.rhoL

        max_abserror = np.max(np.abs((fit_value/EOS_value)-1)*100)

#        import matplotlib.pyplot as plt
#        plt.plot(x,fit_value,x,EOS_value)
#        plt.show()

        chromo.fitness = max_abserror
        chromo.sum_square = myoutput.sum_square
        chromo.max_abserror = max_abserror

        print '.',

        return chromo.fitness

    def tourny_select_chromo(self, samples):
        '''
        Randomly select two chromosomes from the samples, then return the one
        with the best fitness.
        '''
        a = random.choice(samples)
        b = random.choice(samples)
        if a.fitness < b.fitness:
            return a
        else:
            return b

    def breed(self, a, b):
        '''
        Breed two chromosomes by splicing them in a random spot and combining
        them together to form two new chromos.
        '''
        splice_pos = random.randrange(len(a.v))
        new_a = a.v[:splice_pos] + b.v[splice_pos:]
        new_b = b.v[:splice_pos] + a.v[splice_pos:]
        return Sample(sorted(new_a)), Sample(sorted(new_b))

    def mutate(self, chromo):
        '''
        Mutate a chromosome by changing one of the parameters, but only if it improves the fitness
        '''
        v = chromo.v
        if hasattr(chromo,'fitness'):
            old_fitness = chromo.fitness
        else:
            old_fitness = self.fitness(chromo)

        for i in range(10):
            pos = random.randrange(len(chromo.v))
            chromo.v[pos] = random.choice(LIBRARY)
            new_fitness = self.fitness(chromo)
            if new_fitness < old_fitness:
                return chromo
            else:
                return Sample(sorted(v))

    def run(self):
        # Create a random sample of chromos
        samples = self.generate_random_chromosomes()

        # Calculate the fitness for the initial chromosomes
        for chromo in samples:
            self.fitness(chromo)
        print '#'

        decorated = [(sample.fitness,sample) for sample in samples]
        decorated.sort()
        samples = [s for sv,s in decorated]
        values = [sv for sv,s in decorated]
        plt.plot(values[0:len(values)//2])
        plt.show()

        # Main loop: each generation select a subset of the sample and breed from
        # them.
        generation = -1
        while generation < 0 or samples[0].fitness > 0.02 or generation < 3 and generation < 15:
            generation += 1

            # Generate the selected group from sample- take the top 10% of samples
            # and tourny select to generate the rest of selected.
            ten_percent = int(len(samples)*.1)
            selected = samples[:ten_percent]
            while len(selected) < self.num_selected:
                selected.append(self.tourny_select_chromo(samples))

            # Generate the solution group by breeding random chromos from selected
            solution = []
            while len(solution) < self.num_samples:
                solution.extend(self.breed(random.choice(selected),
                                         random.choice(selected)))

            # Apply a mutation to a subset of the solution set
            mutate_indices = random.sample(range(len(solution)), len(solution)//self.mutation_factor)
            for i in mutate_indices:
                solution[i] = self.mutate(solution[i])

            for chromo in solution:
                self.fitness(chromo)
            print '#'

            decorated = [(sample.fitness,sample) for sample in solution]
            decorated.sort()
            samples = [s for sv,s in decorated]

            print '------------------  Top 10 values  ---------------'
            for sample in samples[0:10]:
                print sample.v, sample.fitness, sample.max_abserror

            print samples[0].v, list(samples[0].beta)

            # Print useful stats about this generation
            (min, median, max) =  [samples[0].fitness, samples[len(samples)//2].fitness, samples[-1].fitness]
            print("{0} best value: {1}. fitness: best {2}, median {3}, worst {4}".format(generation, samples[0].v, min, median, max))

        return samples[0]

def main():
    gaf = GeneticAncillaryFitter()
    r = gaf.run()

    print r.v
    print list(r.beta)

if __name__ == "__main__":
  main() <|endoftext|>


<commit_before>from __future__ import absolute_import
from __future__ import print_function

import os
import sys
import ast

from collections import namedtuple

# TODO: Ideally this module creates a simple AST that is mapped into the type
#       domain by coretypes.py
from . import coretypes as T
from .traits import registry, lookup

from ply import lex, yacc
from blaze.plyhacks import yaccfrom, lexfrom
from blaze.error import CustomSyntaxError, BlazeTypeError

#------------------------------------------------------------------------
# Errors
#------------------------------------------------------------------------

class DatashapeSyntaxError(CustomSyntaxError):
    pass

#------------------------------------------------------------------------
# Lexer
#------------------------------------------------------------------------

tokens = (
    'TYPE', 'NAME', 'NUMBER', 'STRING', 'ELLIPSIS', 'EQUALS',
    'COMMA', 'COLON', 'LBRACE', 'RBRACE', 'SEMI', 'BIT',
    'VAR', 'JSON', 'DATA', 'ARROW', 'LBRACK', 'RBRACK',
)

literals = [
    '=' ,
    '|' ,
    ',' ,
    '(' ,
    ')' ,
    ':' ,
    '{' ,
    '}' ,
    '*' ,
    '\!' ,
]

bits = set([
    'bool',
    'bytes',
    'int',
    'float',
    'int8',
    'int16',
    'int32',
    'int64',
    'int64',
    'uint8',
    'uint16',
    'uint32',
    'uint64',
    'float16',
    'float32',
    'float64',
    'float128',
    'cfloat32',
    'cfloat64',
    'complex64',
    'complex128',
    'string',
    'datetime64',
    'timedelta64',
])

t_EQUALS    = r'='
t_COMMA     = r','
t_COLON     = r':'
t_SEMI      = r';'
t_LBRACE    = r'\{'
t_RBRACE    = r'\}'
t_ELLIPSIS  = r'\.\.\.'
t_ARROW     = r'->'
t_LBRACK    = r'\['
t_RBRACK    = r'\]'
t_ignore    = ' '

def t_TYPE(t):
    r'type'
    return t

def t_DATA(t):
    r'data'
    return t

def t_newline(t):
    r'\n+'
    #t.lexer.lineno += t.value.count("\n")

def t_JSON(t):
    # Must be before NAME to match
    r'json'
    return t

def t_VAR(t):
    # Must be before NAME to match
    r'var'
    return t

def t_NAME(t):
    r'[a-zA-Z_][a-zA-Z0-9_]*'
    if t.value in bits:
        t.type = 'BIT'
    return t

def t_COMMENT(t):
    r'\#.*'
    pass

def t_NUMBER(t):
    r'\d+'
    t.value = int(t.value)
    return t

def t_STRING(t):
    r'(?:"(?:[^"\n\r\\]|(?:\\x[0-9a-fA-F]{2})|(?:\\u[0-9a-fA-F]{4})|(?:\\.))*")|(?:\'(?:[^\'\n\r\\]|(?:\\x[0-9a-fA-F]+)|(?:\\u[0-9a-fA-F]{4})|(?:\\.))*\')'
    # Use the Python parser via the ast module to parse the string,
    # since neither the string_escape nor unicode_escape do the right thing
    if sys.version_info >= (3, 0):
        t.value = ast.parse(t.value).body[0].value.s
    else:
        t.value = ast.parse('u' + t.value).body[0].value.s
    return t

def t_error(t):
    raise Exception("Unknown token %s" % repr(t.value[0]))
    #t.lexer.skip(1)

#------------------------------------------------------------------------
# Parser
#------------------------------------------------------------------------

precedence = (
    ('right' , 'COMMA'),
)

dtdecl = namedtuple('dtdecl', 'name, elts')
tydecl = namedtuple('tydecl', 'lhs, rhs')
simpletype = namedtuple('simpletype', 'nargs, tycon, tyvars')

def p_top(p):
    '''top : mod '''
    p[0] = p[1]

#------------------------------------------------------------------------

def p_decl1(p):
    'mod : mod mod'
    p[0] = [p[1], p[2]]

def p_decl2(p):
    'mod : stmt'
    p[0] = p[1]

#------------------------------------------------------------------------

def p_data_assign(p):
    'stmt : DATA NAME EQUALS data'
    p[0] = dtdecl(p[2], p[4])

def p_statement_assign(p):
    'stmt : TYPE lhs_expression EQUALS rhs_expression'

    # alias
    if len(p[2]) == 1:
        constructid = p[2][0]
        parameters  = ()
        rhs         = p[4]

    # parameterized
    else:
        constructid = p[2][0]
        parameters  = p[2][1:]
        rhs         = p[4]

    lhs = simpletype(len(parameters), constructid, parameters)
    p[0] = tydecl(lhs, rhs)

def p_statement_expr(p):
    'stmt : rhs_expression'
    p[0] = p[1]

def p_enum_def1(p):
    "data : data '|' data"
    p[0] = p[1] + p[3]

def p_enum_def3(p):
    'data : NAME'
    p[0] = [p[1]]

#------------------------------------------------------------------------

def p_lhs_expression(p):
    'lhs_expression : lhs_expression lhs_expression'
    # tuple addition
    p[0] = p[1] + p[2]

def p_lhs_expression_node(p):
    'lhs_expression : NAME'
    p[0] = (p[1],)

#------------------------------------------------------------------------

def p_rhs_expression(p):
    '''rhs_expression : rhs_expr
                      | rhs_function
    '''
    p[0] = p[1]

def p_rhs_expr(p):
    '''rhs_expr : rhs_expression_list'''
    if len(p[1]) == 1:
        rhs = p[1][0]
        # if getattr(rhs, 'cls', T.MEASURE) != T.MEASURE:
        #     raise TypeError(
        #         'Only a measure can appear on the last position of a '
        #         'datashape, not %s' % repr(rhs))
        p[0] = rhs
    else:
        p[0] = T.DataShape(*p[1])

def p_rhs_expression_list_node1(p):
    '''rhs_expression_list : appl
                           | record
                           | ctor
    '''
    p[0] = (p[1],)

def p_rhs_expression_list__bit(p):
    '''rhs_expression_list : BIT'''
    p[0] = (T.Type._registry[p[1]],)

def p_rhs_expression_list__name(p):
    '''rhs_expression_list : typevar'''
    p[0] = (p[1],)

def p_rhs_expression_list__contrained(p):
    '''rhs_expression_list : typevar COLON NAME'''
    # Note: This syntax is constrained to type variables only
    typevar = p[1]
    try:
        typeset = registry[p[3]]
    except KeyError:
        raise BlazeTypeError("No typeset '%s' registered" % (p[3],))
    p[0] = (T.Implements(typevar, typeset),)

def p_rhs_expression_list__number(p):
    '''rhs_expression_list : NUMBER'''
    p[0] = (T.Fixed(p[1]),)

def p_rhs_expression_list__var(p):
    '''rhs_expression_list : VAR'''
    p[0] = (T.Var(),)

def p_rhs_expression_list__json(p):
    '''rhs_expression_list : JSON'''
    p[0] = (T.JSON(),)

def p_rhs_expression_list__wild(p):
    '''rhs_expression_list : ELLIPSIS'''
    p[0] = (T.Ellipsis(),)

def p_rhs_expression_list__wild_constrained(p):
    '''rhs_expression_list : typevar ELLIPSIS'''
    p[0] = (T.Ellipsis(p[1]),)

def p_rhs_expression_list(p):
    'rhs_expression_list : rhs_expression_list COMMA rhs_expression_list metadata'
    # tuple addition
    p[0] = p[1] + p[3]

#------------------------------------------------------------------------

def p_rhs_function(p):
    'rhs_function : rhs_signature'
    p[0] = T.Function(*p[1])

def p_rhs_signature(p):
    '''
    rhs_signature : rhs_signature ARROW rhs_signature
    '''
    p[0] = p[1] + p[3]

def p_rhs_signature2(p):
    'rhs_signature : rhs_expr'
    p[0] = [p[1]]

#------------------------------------------------------------------------

def p_typevar(p):
    "typevar : NAME"
    p[0] = T.TypeVar(p[1])

#------------------------------------------------------------------------

def p_metadata1(p):
    "metadata : '!' LBRACE appl_args RBRACE "
    p[0] = p[3]

def p_metadata2(p):
    "metadata : empty"
    p[0] = None

#------------------------------------------------------------------------

def p_appl_args__appl__record(p):
    '''appl_args : appl
                 | record'''
    p[0] = (p[1],)

def p_appl_args__rhs_expression(p):
    "appl_args : '(' rhs_expression ')'"
    p[0] = (p[2],)

def p_appl_args__name(p):
    '''appl_args : typevar'''
    p[0] = (p[1],)

def p_appl_args__bit(p):
    '''appl_args : BIT'''
    p[0] = (T.Type._registry[p[1]],)

def p_appl_args__number(p):
    '''appl_args : NUMBER'''
    p[0] = (T.IntegerConstant(p[1]),)

def p_appl_args__string(p):
    '''appl_args : STRING'''
    p[0] = (T.StringConstant(p[1]),)

def p_appl_args(p):
    'appl_args : appl_args COMMA appl_args'
    # tuple addition
    p[0] = p[1] + p[3]

#------------------------------------------------------------------------

def p_appl(p):
    """appl : NAME '(' appl_args ')'
            | BIT '(' appl_args ')'""" # BIT is here for 'string(...)'

    if p[1] == 'Categorical': # TODO: don't hardcode
        if not all(isinstance(x, T.TypeVar) for x in p[3]):
            raise Exception('Invalid categorical definition')

        p[0] = T.Enum(None, *p[3])
    if p[1] in reserved:
        # The appl_args part of the grammar already produces
        # TypeVar/IntegerConstant/StringConstant values
        p[0] = reserved[p[1]](*p[3])
    else:
        raise NameError('Cannot use the name %s for type application' % repr(p[1]))

#------------------------------------------------------------------------

def p_ctor(p):
    """ctor : NAME LBRACK rhs_expression RBRACK"""

    # Application of a type constructor. Square brackets are more apt for
    # type constructor application than parentheses when types are implemented
    # as Python classes (e.g. in Numba). Parentheses then instantiate the type
    # to a value, and square brackets parameterize the type.

    name = p[1]
    args = p[3]

    n = len(args)
    flags = [{'coercible': False} for i in range(n)]
    ctor = T.TypeConstructor(name, n, flags)
    p[0] = ctor(*args)

#------------------------------------------------------------------------

def p_record(p):
    'record : LBRACE record_opt RBRACE'
    p[0] = T.Record(p[2])

def p_record_opt1(p):
    'record_opt : record_opt SEMI record_opt'
    p[0] = p[1] + p[3]

def p_record_opt2(p):
    'record_opt : record_item'
    p[0] = [p[1]]

def p_record_opt3(p):
    'record_opt : empty'
    p[0] = []

def p_record_name(p):
    '''record_name : NAME
                   | BIT
                   | TYPE
                   | JSON
                   | VAR'''
    p[0] = p[1]

def p_record_item1(p):
    "record_item : record_name COLON '(' rhs_expression ')'"
    p[0] = (p[1], p[4])

def p_record_item2(p):
    '''record_item : record_name COLON rhs_expression'''
    p[0] = (p[1], p[3])

#------------------------------------------------------------------------

def p_empty(t):
    'empty : '
    pass

def p_error(p):
    if p:
        raise DatashapeSyntaxError(
            p.lexpos,
            '<stdin>',
            p.lexer.lexdata,
        )
    else:
        raise DatashapeSyntaxError(
            0,
            '<stdin>',
            '',
        )

#------------------------------------------------------------------------
# Toplevel
#------------------------------------------------------------------------

reserved = {
    'Record'      : T.Record,
    'Range'       : T.Range,
    'Categorical' : T.Enum,
    'Option'      : T.Option,
    #'Either'   : T.Either,
    #'Union'    : T.Union,
    'string'   : T.String, # String type per proposal
}

def debug_parse(data, lexer, parser):
    lexer.input(data)
    while True:
        tok = lexer.token()
        if not tok: break
        print(tok)

    import logging
    logging.basicConfig(
        level = logging.DEBUG,
        filename = "parselog.txt",
        filemode = "w",
        format = "%(filename)10s:%(lineno)4d:%(message)s"
    )
    log = logging.getLogger()
    return parser.parse(data, debug=log)

#------------------------------------------------------------------------
# Toplevel
#------------------------------------------------------------------------

def rebuild():
    """ Rebuild the parser and lexer tables. """
    path = os.path.abspath(__file__)
    output = os.path.dirname(path)
    module = sys.modules[__name__]

    lex.lex(module=module, lextab="dlex", outputdir=output, debug=0, optimize=1)
    yacc.yacc(tabmodule='dyacc',outputdir=output, write_tables=1, debug=0, optimize=1)

    sys.stdout.write("Parse and lexer tables rebuilt.\n")

def _parse(source):
    try:
        from . import dlex
        from . import dyacc
    except ImportError:
        rebuild()
        from . import dlex
        from . import dyacc

    module = sys.modules[__name__]
    lexer  = lexfrom(module, dlex)
    parser = yaccfrom(module, dyacc, lexer)

    return parser.parse(source, lexer=lexer)

def parse_mod(pattern):
    dss = _parse(pattern)

    for ds in dss:
        if isinstance(ds, tydecl):
            yield ds.rhs
        elif isinstance(ds, dtdecl):
            yield T.Enum(ds.name, ds.elts)

def parse(pattern):
    ds = _parse(pattern)

    if isinstance(ds, dtdecl):
        raise TypeError('Predeclared categorical types are not allowed inline')

    # Just take the type from "type X = Y" statements
    if isinstance(ds, tydecl):
        if ds.lhs.nargs == 0:
            ds = ds.rhs
        else:
            raise TypeError('building a simple dshape with '
                            'type parameters is not supported')
    # Require that the type be concrete, not parameterized
    if isinstance(ds, T.TypeVar):
        raise TypeError(('Only a measure can appear on the last '
                        'position of a datashape, not %s') % repr(ds))
    return ds

if __name__ == '__main__':
    import readline
    # build the parse tablr
    rebuild()

    if len(sys.argv) > 1:
        ds_mod = open(sys.argv[1]).read()
        ast = list(parse_mod(ds_mod))
        print(ast)
    else:
        readline.parse_and_bind('')
        while True:
            try:
                line = raw_input('>> ')
                ast = parse(line)
                print(ast)
            except EOFError:
                break
<commit_msg>Make sure the parser is rebuild properly<commit_after>from __future__ import absolute_import
from __future__ import print_function

import os
import sys
import ast

from collections import namedtuple

# TODO: Ideally this module creates a simple AST that is mapped into the type
#       domain by coretypes.py
from . import coretypes as T
from .traits import registry, lookup

from ply import lex, yacc
from blaze.plyhacks import yaccfrom, lexfrom
from blaze.error import CustomSyntaxError, BlazeTypeError

#------------------------------------------------------------------------
# Errors
#------------------------------------------------------------------------

class DatashapeSyntaxError(CustomSyntaxError):
    pass

#------------------------------------------------------------------------
# Lexer
#------------------------------------------------------------------------

tokens = (
    'TYPE', 'NAME', 'NUMBER', 'STRING', 'ELLIPSIS', 'EQUALS',
    'COMMA', 'COLON', 'LBRACE', 'RBRACE', 'SEMI', 'BIT',
    'VAR', 'JSON', 'DATA', 'ARROW', 'LBRACK', 'RBRACK',
)

literals = [
    '=' ,
    '|' ,
    ',' ,
    '(' ,
    ')' ,
    ':' ,
    '{' ,
    '}' ,
    '*' ,
    '\!' ,
]

bits = set([
    'bool',
    'bytes',
    'int',
    'float',
    'int8',
    'int16',
    'int32',
    'int64',
    'int64',
    'uint8',
    'uint16',
    'uint32',
    'uint64',
    'float16',
    'float32',
    'float64',
    'float128',
    'cfloat32',
    'cfloat64',
    'complex64',
    'complex128',
    'string',
    'datetime64',
    'timedelta64',
])

t_EQUALS    = r'='
t_COMMA     = r','
t_COLON     = r':'
t_SEMI      = r';'
t_LBRACE    = r'\{'
t_RBRACE    = r'\}'
t_ELLIPSIS  = r'\.\.\.'
t_ARROW     = r'->'
t_LBRACK    = r'\['
t_RBRACK    = r'\]'
t_ignore    = ' '

def t_TYPE(t):
    r'type'
    return t

def t_DATA(t):
    r'data'
    return t

def t_newline(t):
    r'\n+'
    #t.lexer.lineno += t.value.count("\n")

def t_JSON(t):
    # Must be before NAME to match
    r'json'
    return t

def t_VAR(t):
    # Must be before NAME to match
    r'var'
    return t

def t_NAME(t):
    r'[a-zA-Z_][a-zA-Z0-9_]*'
    if t.value in bits:
        t.type = 'BIT'
    return t

def t_COMMENT(t):
    r'\#.*'
    pass

def t_NUMBER(t):
    r'\d+'
    t.value = int(t.value)
    return t

def t_STRING(t):
    r'(?:"(?:[^"\n\r\\]|(?:\\x[0-9a-fA-F]{2})|(?:\\u[0-9a-fA-F]{4})|(?:\\.))*")|(?:\'(?:[^\'\n\r\\]|(?:\\x[0-9a-fA-F]+)|(?:\\u[0-9a-fA-F]{4})|(?:\\.))*\')'
    # Use the Python parser via the ast module to parse the string,
    # since neither the string_escape nor unicode_escape do the right thing
    if sys.version_info >= (3, 0):
        t.value = ast.parse(t.value).body[0].value.s
    else:
        t.value = ast.parse('u' + t.value).body[0].value.s
    return t

def t_error(t):
    raise Exception("Unknown token %s" % repr(t.value[0]))
    #t.lexer.skip(1)

#------------------------------------------------------------------------
# Parser
#------------------------------------------------------------------------

precedence = (
    ('right' , 'COMMA'),
)

dtdecl = namedtuple('dtdecl', 'name, elts')
tydecl = namedtuple('tydecl', 'lhs, rhs')
simpletype = namedtuple('simpletype', 'nargs, tycon, tyvars')

def p_top(p):
    '''top : mod '''
    p[0] = p[1]

#------------------------------------------------------------------------

def p_decl1(p):
    'mod : mod mod'
    p[0] = [p[1], p[2]]

def p_decl2(p):
    'mod : stmt'
    p[0] = p[1]

#------------------------------------------------------------------------

def p_data_assign(p):
    'stmt : DATA NAME EQUALS data'
    p[0] = dtdecl(p[2], p[4])

def p_statement_assign(p):
    'stmt : TYPE lhs_expression EQUALS rhs_expression'

    # alias
    if len(p[2]) == 1:
        constructid = p[2][0]
        parameters  = ()
        rhs         = p[4]

    # parameterized
    else:
        constructid = p[2][0]
        parameters  = p[2][1:]
        rhs         = p[4]

    lhs = simpletype(len(parameters), constructid, parameters)
    p[0] = tydecl(lhs, rhs)

def p_statement_expr(p):
    'stmt : rhs_expression'
    p[0] = p[1]

def p_enum_def1(p):
    "data : data '|' data"
    p[0] = p[1] + p[3]

def p_enum_def3(p):
    'data : NAME'
    p[0] = [p[1]]

#------------------------------------------------------------------------

def p_lhs_expression(p):
    'lhs_expression : lhs_expression lhs_expression'
    # tuple addition
    p[0] = p[1] + p[2]

def p_lhs_expression_node(p):
    'lhs_expression : NAME'
    p[0] = (p[1],)

#------------------------------------------------------------------------

def p_rhs_expression(p):
    '''rhs_expression : rhs_expr
                      | rhs_function
    '''
    p[0] = p[1]

def p_rhs_expr(p):
    '''rhs_expr : rhs_expression_list'''
    if len(p[1]) == 1:
        rhs = p[1][0]
        # if getattr(rhs, 'cls', T.MEASURE) != T.MEASURE:
        #     raise TypeError(
        #         'Only a measure can appear on the last position of a '
        #         'datashape, not %s' % repr(rhs))
        p[0] = rhs
    else:
        p[0] = T.DataShape(*p[1])

def p_rhs_expression_list_node1(p):
    '''rhs_expression_list : appl
                           | record
                           | ctor
    '''
    p[0] = (p[1],)

def p_rhs_expression_list__bit(p):
    '''rhs_expression_list : BIT'''
    p[0] = (T.Type._registry[p[1]],)

def p_rhs_expression_list__name(p):
    '''rhs_expression_list : typevar'''
    p[0] = (p[1],)

def p_rhs_expression_list__contrained(p):
    '''rhs_expression_list : typevar COLON NAME'''
    # Note: This syntax is constrained to type variables only
    typevar = p[1]
    try:
        typeset = registry[p[3]]
    except KeyError:
        raise BlazeTypeError("No typeset '%s' registered" % (p[3],))
    p[0] = (T.Implements(typevar, typeset),)

def p_rhs_expression_list__number(p):
    '''rhs_expression_list : NUMBER'''
    p[0] = (T.Fixed(p[1]),)

def p_rhs_expression_list__var(p):
    '''rhs_expression_list : VAR'''
    p[0] = (T.Var(),)

def p_rhs_expression_list__json(p):
    '''rhs_expression_list : JSON'''
    p[0] = (T.JSON(),)

def p_rhs_expression_list__wild(p):
    '''rhs_expression_list : ELLIPSIS'''
    p[0] = (T.Ellipsis(),)

def p_rhs_expression_list__wild_constrained(p):
    '''rhs_expression_list : typevar ELLIPSIS'''
    p[0] = (T.Ellipsis(p[1]),)

def p_rhs_expression_list(p):
    'rhs_expression_list : rhs_expression_list COMMA rhs_expression_list metadata'
    # tuple addition
    p[0] = p[1] + p[3]

#------------------------------------------------------------------------

def p_rhs_function(p):
    'rhs_function : rhs_signature'
    p[0] = T.Function(*p[1])

def p_rhs_signature(p):
    '''
    rhs_signature : rhs_signature ARROW rhs_signature
    '''
    p[0] = p[1] + p[3]

def p_rhs_signature2(p):
    'rhs_signature : rhs_expr'
    p[0] = [p[1]]

#------------------------------------------------------------------------

def p_typevar(p):
    "typevar : NAME"
    p[0] = T.TypeVar(p[1])

#------------------------------------------------------------------------

def p_metadata1(p):
    "metadata : '!' LBRACE appl_args RBRACE "
    p[0] = p[3]

def p_metadata2(p):
    "metadata : empty"
    p[0] = None

#------------------------------------------------------------------------

def p_appl_args__appl__record(p):
    '''appl_args : appl
                 | record'''
    p[0] = (p[1],)

def p_appl_args__rhs_expression(p):
    "appl_args : '(' rhs_expression ')'"
    p[0] = (p[2],)

def p_appl_args__name(p):
    '''appl_args : typevar'''
    p[0] = (p[1],)

def p_appl_args__bit(p):
    '''appl_args : BIT'''
    p[0] = (T.Type._registry[p[1]],)

def p_appl_args__number(p):
    '''appl_args : NUMBER'''
    p[0] = (T.IntegerConstant(p[1]),)

def p_appl_args__string(p):
    '''appl_args : STRING'''
    p[0] = (T.StringConstant(p[1]),)

def p_appl_args(p):
    'appl_args : appl_args COMMA appl_args'
    # tuple addition
    p[0] = p[1] + p[3]

#------------------------------------------------------------------------

def p_appl(p):
    """appl : NAME '(' appl_args ')'
            | BIT '(' appl_args ')'""" # BIT is here for 'string(...)'

    if p[1] == 'Categorical': # TODO: don't hardcode
        if not all(isinstance(x, T.TypeVar) for x in p[3]):
            raise Exception('Invalid categorical definition')

        p[0] = T.Enum(None, *p[3])
    if p[1] in reserved:
        # The appl_args part of the grammar already produces
        # TypeVar/IntegerConstant/StringConstant values
        p[0] = reserved[p[1]](*p[3])
    else:
        raise NameError('Cannot use the name %s for type application' % repr(p[1]))

#------------------------------------------------------------------------

def p_ctor(p):
    """ctor : NAME LBRACK rhs_expression RBRACK"""

    # Application of a type constructor. Square brackets are more apt for
    # type constructor application than parentheses when types are implemented
    # as Python classes (e.g. in Numba). Parentheses then instantiate the type
    # to a value, and square brackets parameterize the type.

    name = p[1]
    args = p[3]

    n = len(args)
    flags = [{'coercible': False} for i in range(n)]
    ctor = T.TypeConstructor(name, n, flags)
    p[0] = ctor(*args)

#------------------------------------------------------------------------

def p_record(p):
    'record : LBRACE record_opt RBRACE'
    p[0] = T.Record(p[2])

def p_record_opt1(p):
    'record_opt : record_opt SEMI record_opt'
    p[0] = p[1] + p[3]

def p_record_opt2(p):
    'record_opt : record_item'
    p[0] = [p[1]]

def p_record_opt3(p):
    'record_opt : empty'
    p[0] = []

def p_record_name(p):
    '''record_name : NAME
                   | BIT
                   | TYPE
                   | JSON
                   | VAR'''
    p[0] = p[1]

def p_record_item1(p):
    "record_item : record_name COLON '(' rhs_expression ')'"
    p[0] = (p[1], p[4])

def p_record_item2(p):
    '''record_item : record_name COLON rhs_expression'''
    p[0] = (p[1], p[3])

#------------------------------------------------------------------------

def p_empty(t):
    'empty : '
    pass

def p_error(p):
    if p:
        raise DatashapeSyntaxError(
            p.lexpos,
            '<stdin>',
            p.lexer.lexdata,
        )
    else:
        raise DatashapeSyntaxError(
            0,
            '<stdin>',
            '',
        )

#------------------------------------------------------------------------
# Toplevel
#------------------------------------------------------------------------

reserved = {
    'Record'      : T.Record,
    'Range'       : T.Range,
    'Categorical' : T.Enum,
    'Option'      : T.Option,
    #'Either'   : T.Either,
    #'Union'    : T.Union,
    'string'   : T.String, # String type per proposal
}

def debug_parse(data, lexer, parser):
    lexer.input(data)
    while True:
        tok = lexer.token()
        if not tok: break
        print(tok)

    import logging
    logging.basicConfig(
        level = logging.DEBUG,
        filename = "parselog.txt",
        filemode = "w",
        format = "%(filename)10s:%(lineno)4d:%(message)s"
    )
    log = logging.getLogger()
    return parser.parse(data, debug=log)

#------------------------------------------------------------------------
# Toplevel
#------------------------------------------------------------------------

def rebuild():
    """ Rebuild the parser and lexer tables. """
    path = os.path.abspath(__file__)
    output = os.path.dirname(path)
    module = sys.modules[__name__]

    lex.lex(module=module, lextab="dlex", outputdir=output, debug=0, optimize=1)
    yacc.yacc(tabmodule='dyacc',outputdir=output, write_tables=1, debug=0, optimize=1)

    sys.stdout.write("Parse and lexer tables rebuilt.\n")

def _parse(source):
    try:
        from . import dlex
        from . import dyacc
    except ImportError:
        rebuild()
        from . import dlex
        from . import dyacc

    module = sys.modules[__name__]
    lexer  = lexfrom(module, dlex)
    parser = yaccfrom(module, dyacc, lexer)

    return parser.parse(source, lexer=lexer)

def parse_mod(pattern):
    dss = _parse(pattern)

    for ds in dss:
        if isinstance(ds, tydecl):
            yield ds.rhs
        elif isinstance(ds, dtdecl):
            yield T.Enum(ds.name, ds.elts)

def parse(pattern):
    ds = _parse(pattern)

    if isinstance(ds, dtdecl):
        raise TypeError('Predeclared categorical types are not allowed inline')

    # Just take the type from "type X = Y" statements
    if isinstance(ds, tydecl):
        if ds.lhs.nargs == 0:
            ds = ds.rhs
        else:
            raise TypeError('building a simple dshape with '
                            'type parameters is not supported')
    # Require that the type be concrete, not parameterized
    if isinstance(ds, T.TypeVar):
        raise TypeError(('Only a measure can appear on the last '
                        'position of a datashape, not %s') % repr(ds))
    return ds

if __name__ == '__main__':
    import readline
    # build the parse tablr
    rebuild()

    if len(sys.argv) > 1:
        ds_mod = open(sys.argv[1]).read()
        ast = list(parse_mod(ds_mod))
        print(ast)
    else:
        readline.parse_and_bind('')
        while True:
            try:
                line = raw_input('>> ')
                ast = parse(line)
                print(ast)
            except EOFError:
                break

if int(os.environ.get('BLAZE_REBUILD_PARSER', 0)):
    # Rebuild the parser before it is used anywhere
    rebuild()<|endoftext|>


<commit_before>from django import forms
from django.conf import settings
from django.forms import inlineformset_factory
from django.utils.translation import ugettext_lazy as _

from adhocracy4.categories import models as category_models
from adhocracy4.dashboard.components.forms import ModuleDashboardFormSet
from adhocracy4.modules import models as module_models

from .widgets import SelectWithIconWidget

def has_icons(module):
    if not hasattr(settings, 'A4_CATEGORY_ICONS'):
        return False

    module_settings = module.settings_instance
    return module_settings and hasattr(module_settings, 'polygon')

class CategorizableFieldMixin:
    category_field_name = 'category'

    def __init__(self, *args, **kwargs):
        module = kwargs.pop('module')
        super().__init__(*args, **kwargs)

        field = self.fields[self.category_field_name]
        field.queryset = category_models.Category.objects.filter(module=module)

        required = field.queryset.exists()
        field.empty_label = None
        field.required = required
        field.widget.is_required = required

    def show_categories(self):
        field = self.fields[self.category_field_name]
        module_has_categories = field.queryset.exists()
        return module_has_categories

class CategoryForm(forms.ModelForm):
    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)

        instance = kwargs.get('instance', None)
        if not (instance and has_icons(instance.module)):
            del self.fields['icon']

    name = forms.CharField(widget=forms.TextInput(attrs={
        'placeholder': _('Category')}
    ))

    class Media:
        js = ('js/formset.js',)

    class Meta:
        model = category_models.Category
        fields = ['name', 'icon']

        widgets = {
            'icon': SelectWithIconWidget,
        }

CategoryFormSet = inlineformset_factory(module_models.Module,
                                        category_models.Category,
                                        form=CategoryForm,
                                        formset=ModuleDashboardFormSet,
                                        extra=0,
                                        )
<commit_msg>Pass module to form to be able to recognize mapy modules<commit_after>from django import forms
from django.conf import settings
from django.forms import inlineformset_factory
from django.utils.translation import ugettext_lazy as _

from adhocracy4.categories import models as category_models
from adhocracy4.dashboard.components.forms import ModuleDashboardFormSet
from adhocracy4.modules import models as module_models

from .widgets import SelectWithIconWidget

def has_icons(module):
    if not hasattr(settings, 'A4_CATEGORY_ICONS'):
        return False

    module_settings = module.settings_instance
    return module_settings and hasattr(module_settings, 'polygon')

class CategorizableFieldMixin:
    category_field_name = 'category'

    def __init__(self, *args, **kwargs):
        module = kwargs.pop('module')
        super().__init__(*args, **kwargs)

        field = self.fields[self.category_field_name]
        field.queryset = category_models.Category.objects.filter(module=module)

        required = field.queryset.exists()
        field.empty_label = None
        field.required = required
        field.widget.is_required = required

    def show_categories(self):
        field = self.fields[self.category_field_name]
        module_has_categories = field.queryset.exists()
        return module_has_categories

class CategoryForm(forms.ModelForm):
    def __init__(self, module, *args, **kwargs):
        super().__init__(*args, **kwargs)

        if not (module and has_icons(module)):
            del self.fields['icon']

    name = forms.CharField(widget=forms.TextInput(attrs={
        'placeholder': _('Category')}
    ))

    class Media:
        js = ('js/formset.js',)

    class Meta:
        model = category_models.Category
        fields = ['name', 'icon']

        widgets = {
            'icon': SelectWithIconWidget,
        }

class CategoryModuleDashboardFormSet(ModuleDashboardFormSet):
    def get_form_kwargs(self, index):
        form_kwargs = super().get_form_kwargs(index)
        form_kwargs['module'] = self.instance
        return form_kwargs

CategoryFormSet = inlineformset_factory(module_models.Module,
                                        category_models.Category,
                                        form=CategoryForm,
                                        formset=CategoryModuleDashboardFormSet,
                                        extra=0,
                                        )
<|endoftext|>


<commit_before># Copyright 2013-2021 Lawrence Livermore National Security, LLC and other
# Spack Project Developers. See the top-level COPYRIGHT file for details.
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)

from spack import *

class RHmisc(RPackage):
    """Contains many functions useful for data analysis, high-level
    graphics, utility operations, functions for computing sample size
    and power, importing and annotating datasets, imputing missing
    values, advanced table making, variable clustering, character
    string manipulation, conversion of R objects to LaTeX and html
    code, and recoding variables."""

    homepage = "http://biostat.mc.vanderbilt.edu/Hmisc"
    url      = "https://cloud.r-project.org/src/contrib/Hmisc_4.1-1.tar.gz"
    list_url = "https://cloud.r-project.org/src/contrib/Archive/Hmisc"

    version('4.4-0', sha256='f16ecf4c5ee2202d51f426282a54f8000ffa8b9747c3e910205f34f878556ec7')
    version('4.2-0', sha256='9e9614673288dd00295f250fa0bf96fc9e9fed692c69bf97691081c1a01411d9')
    version('4.1-1', sha256='991db21cdf73ffbf5b0239a4876b2e76fd243ea33528afd88dc968792f281498')

    depends_on('r-lattice', type=('build', 'run'))
    depends_on('r-survival@2.40-1:', type=('build', 'run'))
    depends_on('r-survival@3.1-6:', when='@4.4:', type=('build', 'run'))
    depends_on('r-formula', type=('build', 'run'))
    depends_on('r-ggplot2@2.2:', type=('build', 'run'))
    depends_on('r-latticeextra', type=('build', 'run'))
    depends_on('r-acepack', type=('build', 'run'))
    depends_on('r-gridextra', type=('build', 'run'))
    depends_on('r-data-table', type=('build', 'run'))
    depends_on('r-htmltools', type=('build', 'run'))
    depends_on('r-base64enc', type=('build', 'run'))
    depends_on('r-htmltable@1.11.0:', type=('build', 'run'))
    depends_on('r-viridis', type=('build', 'run'))
    depends_on('r-cluster', type=('build', 'run'))
    depends_on('r-rpart', type=('build', 'run'))
    depends_on('r-nnet', type=('build', 'run'))
    depends_on('r-foreign', type=('build', 'run'))
    depends_on('r-gtable', type=('build', 'run'))
<commit_msg>add version 4.4-2 to r-hmisc (#21044)<commit_after># Copyright 2013-2021 Lawrence Livermore National Security, LLC and other
# Spack Project Developers. See the top-level COPYRIGHT file for details.
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)

from spack import *

class RHmisc(RPackage):
    """Harrell Miscellaneous

    Contains many functions useful for data analysis, high-level
    graphics, utility operations, functions for computing sample size
    and power, importing and annotating datasets, imputing missing
    values, advanced table making, variable clustering, character
    string manipulation, conversion of R objects to LaTeX and html
    code, and recoding variables."""

    homepage = "http://biostat.mc.vanderbilt.edu/Hmisc"
    url      = "https://cloud.r-project.org/src/contrib/Hmisc_4.1-1.tar.gz"
    list_url = "https://cloud.r-project.org/src/contrib/Archive/Hmisc"

    version('4.4-2', sha256='490ac64dd8558868e7c6fdd9523af102e17ea536c450d62c48b04155279bfbc8')
    version('4.4-0', sha256='f16ecf4c5ee2202d51f426282a54f8000ffa8b9747c3e910205f34f878556ec7')
    version('4.2-0', sha256='9e9614673288dd00295f250fa0bf96fc9e9fed692c69bf97691081c1a01411d9')
    version('4.1-1', sha256='991db21cdf73ffbf5b0239a4876b2e76fd243ea33528afd88dc968792f281498')

    depends_on('r-lattice', type=('build', 'run'))
    depends_on('r-survival@2.40-1:', type=('build', 'run'))
    depends_on('r-survival@3.1-6:', when='@4.4:', type=('build', 'run'))
    depends_on('r-formula', type=('build', 'run'))
    depends_on('r-ggplot2@2.2:', type=('build', 'run'))
    depends_on('r-latticeextra', type=('build', 'run'))
    depends_on('r-cluster', type=('build', 'run'))
    depends_on('r-rpart', type=('build', 'run'))
    depends_on('r-nnet', type=('build', 'run'))
    depends_on('r-foreign', type=('build', 'run'))
    depends_on('r-gtable', type=('build', 'run'))
    depends_on('r-gridextra', type=('build', 'run'))
    depends_on('r-data-table', type=('build', 'run'))
    depends_on('r-htmltable@1.11.0:', type=('build', 'run'))
    depends_on('r-viridis', type=('build', 'run'))
    depends_on('r-htmltools', type=('build', 'run'))
    depends_on('r-base64enc', type=('build', 'run'))
    depends_on('r-acepack', when='@:4.4-0', type=('build', 'run'))
<|endoftext|>


<commit_before>#!/usr/bin/python

import sys, getopt, exceptions, re, time
import json
from bigram import bigram
from unigram import unigram
from stats import stats
from tools import splitter, generator ,out
from nltk.stem.lancaster import LancasterStemmer

def timeExec(func):
    def wrapper(*arg):
        start = time.clock()
        a = func(*arg)
        end =time.clock()
        print 'used:', end - start
        return a
    return wrapper

def writeFile(name,content):
    print("====== Writing '"+ name +"' to the disk ======")
    output = open(name, 'w')
    output.write(content)
    output.close()
    print("Done")

def createStopWordsList(text):
    stopWordsList = []

    for line in text:
        stopWordsList.extend(line.strip('\n').split(' '))
    return stopWordsList

def extractRawData(text,stopWords = None):
    st = LancasterStemmer()
    rawData = []
    text.readline()
    print("********** Skipped the first line **********")
    print("********** Creating Raw Data **********")

    if stopWords:
        signal = 'ON'
    else:
        signal = 'OFF'

    prevId = 0

    for line in text:

        lineTokens = line.strip('\n').split('\t')
        sentenceId = int(lineTokens[1])

        if sentenceId > prevId:
            prevId = sentenceId
            sentenceStr = lineTokens[2]
            sentiment = int(lineTokens[3])
            sentenceTokens = re.sub("\s+"," ",sentenceStr).split(' ')
            sentenceTokens = map(lambda x:x.lower(),sentenceTokens)
            sentenceTokens = map(lambda x:st.stem(x),sentenceTokens)
            # if stopwords are required, do the following
            if stopWords:
                sentenceTokens = stripWords(sentenceTokens,stopWords)
            entry = {"sentenceId":sentenceId,"sentence":sentenceTokens,"sentiment":sentiment}
            rawData.append(entry)

    print("Done")
    return rawData

#@timeExec
def createInvertedRawData(text, stopWords = None):
    st = LancasterStemmer()

    invertedRawData = {}
    text.readline()
    print("********** Skipped the first line **********")
    print("********** Creating Inverted Raw Data **********")

    if stopWords:
        signal = 'ON'
    else:
        signal = 'OFF'

    print("Strip Stop-words:" + signal)

    for line in text:
        # Split line
        lineTokens = line.strip('\n').split('\t')

        # get PharseId
        pharseId = int(lineTokens[0])

        # get SentenceId
        sentenceId = int(lineTokens[1])

        # get sentence
        sentenceStr = lineTokens[2]

        # get sentiment
        sentiment = int(lineTokens[3])

        # replace one or more spaces by single space
        # then split
        sentenceTokens = re.sub("\s+"," ",sentenceStr).split(' ')

        sentenceTokens = map(lambda x:x.lower(),sentenceTokens)
        sentenceTokens = map(lambda x:st.stem(x),sentenceTokens)
        #print(sentenceTokens)

        # if stopwords are required, do the following
        if stopWords:
            sentenceTokens = stripWords(sentenceTokens,stopWords)

        # create a sentiment key if doesn't exist
        entry = {"pharseId":pharseId,"sentenceId":sentenceId,"sentence":sentenceTokens}

        if sentiment in invertedRawData:
            # append the sentence to its corresponding sentiment list
            invertedRawData[sentiment].append(entry)
        else:
            invertedRawData[sentiment] = []
            invertedRawData[sentiment].append(entry)
    print("Done")
    return invertedRawData

def stripWords(sentenceTokens,wordsList):

    return filter(lambda x: x not in wordsList, sentenceTokens)

def readArgsFromInput(argv):
    rawDataName = stopWordsName = outputName = directory = None
    # Record the flags that user are using
    usedOpts = []
    try:
        opts, args = getopt.getopt(argv,"hr:s:i1o",["rawdata=","stopwords=","ouput="])
    except getopt.GetoptError:
        print 'run parser.py with flag -h to get help doc'
        sys.exit()
    for opt, arg in opts:
        if opt == '-h':
            print 'Flags List:'
            print '-h Help'
            print '-r <rawdata_filename> Import raw data file'
            print '-s <stopwords_filename> Import stopwords file'
            print "-o <outputfile_filename> Define the name of output file.(It's named as no_stopwords.txt by default)"
            print '-1 Mode 1: Process raw data without stripping the stopwords'
            sys.exit()

        elif opt in ("-r","--rawdata"):
            rawDataName = arg
            usedOpts.append('-r')

        elif opt in ("-s","--stopwords"):
            stopWordsName = arg
            usedOpts.append('-s')

        elif opt in ("-o","--output"):
            outputName = arg
            usedOpts.append('-o')

        elif opt in ("-i"):
            usedOpts.append('-i')
            # TODO:export stats images
            pass
        elif opt in ("-1"):
            usedOpts.append('-1')
            # TODO:Mode 1 -- Process raw data without stripping the stopwords
            pass
        else:
            sys.exit("Invalid Flag(s)")

    # pareser wihout flag -o, then the output will be named as this by default
    if '-o' not in usedOpts:
        outputName = 'default_invertedRawdata.json'

    # cannot proceed without filename of raw data
    if rawDataName == None:
        sys.exit("Cannot proceed wihout the filename of raw data")

    return rawDataName, stopWordsName, outputName, usedOpts

def main(argv):

    rawDataName, stopWordsName, outputName, usedOpts = readArgsFromInput(argv)
    # load raw data
    try:
        rawDataFile = open(rawDataName,'r')

        # parser is in mode 1 then prcoess raw data without stripping the stopwords
        if '-1' in usedOpts:
            invertedRawData = createInvertedRawData(rawDataFile)
            rawDataFile.close()

            rawDataFile = open(rawDataName,'r')
            rawData = extractRawData(rawDataFile)
            rawDataFile.close()

        else:
            stopWordsFile = open(stopWordsName,'r')

            # create stopwords list
            stopWordsList = createStopWordsList(stopWordsFile)

            # create invertedRawData
            invertedRawData = createInvertedRawData(rawDataFile,stopWordsList)

            rawDataFile.close()

            rawDataFile = open(rawDataName,'r')

            # extract rawData from rawDataFile
            rawData = extractRawData(rawDataFile,stopWordsList)
            rawDataFile.close()

            average = stats.getWordAverageSentiment(rawData, 2)

            out.saveWordSentiment(average,"average.txt")

            # Now bag of words is ready for feature construction
            matrix2 = generator.createFeatureBagMatrix(rawData)

        # stats.report(invertedRawData)
        # extract the bigrams from inverted raw data
        bigramsInvertedCollection = bigram.extractBigramsFromInvertedRawData(invertedRawData)
        freqDist = bigram.getFrequencyDist(bigramsInvertedCollection)

        # extract the unigrams from inverted raw data
        unigramsInvertedCollection = unigram.extractUnigramsFromInvertedRawData(invertedRawData)
        unifreqDist = bigram.getFrequencyDist(unigramsInvertedCollection)

        # extract the bigrams from raw data
        bigramsRawData = bigram.extractBigramsFromRawData(rawData)

        # bigram sentences by n

        splitedBigramRawData = splitter.splitSentence(3,bigramsRawData)

        matrix = generator.createFreqMatrix(3,splitedBigramRawData,freqDist)

        # dump all the data
        #jsonInvertedRawData = json.dumps(invertedRawData)
        #jsonBigrams = json.dumps(bigramsInvertedCollection)
        #jsonUnigrams = json.dumps(unigramsInvertedCollection)
        #jsonRawData = json.dumps(rawData)
        #jsonSplitedBigramRawData = json.dumps(splitedBigramRawData)

        rawDataFile.close()
        stopWordsFile.close()

        # save files
        #writeFile(outputName,jsonInvertedRawData)
        #writeFile('rawdata.json',jsonRawData)
        #writeFile('SplitedBigramRawData.json',jsonSplitedBigramRawData)
        #writeFile('bigramsCollection.json',jsonBigrams)
        #writeFile('unigramsCollection.json',jsonUnigrams)
    except IOError as e:
        print "Cannot read raw data file or stop-word file"
if __name__ == "__main__":
    main(sys.argv[1:])
<commit_msg>parser<commit_after>#!/usr/bin/python

import sys, getopt, exceptions, re, time
import json
from bigram import bigram
from unigram import unigram
from stats import stats
from tools import splitter, generator ,out
from nltk.stem.lancaster import LancasterStemmer

def timeExec(func):
    def wrapper(*arg):
        start = time.clock()
        a = func(*arg)
        end =time.clock()
        print 'used:', end - start
        return a
    return wrapper

def writeFile(name,content):
    print("====== Writing '"+ name +"' to the disk ======")
    output = open(name, 'w')
    output.write(content)
    output.close()
    print("Done")

def createStopWordsList(text):
    stopWordsList = []

    for line in text:
        stopWordsList.extend(line.strip('\n').split(' '))
    return stopWordsList

def extractRawData(text,stopWords = None):
    st = LancasterStemmer()
    rawData = []
    text.readline()
    print("********** Skipped the first line **********")
    print("********** Creating Raw Data **********")

    if stopWords:
        signal = 'ON'
    else:
        signal = 'OFF'

    prevId = 0

    for line in text:

        lineTokens = line.strip('\n').split('\t')
        sentenceId = int(lineTokens[1])

        if sentenceId > prevId:
            prevId = sentenceId
            sentenceStr = lineTokens[2]
            sentiment = int(lineTokens[3])
            sentenceTokens = re.sub("\s+"," ",sentenceStr).split(' ')
            sentenceTokens = map(lambda x:x.lower(),sentenceTokens)
            sentenceTokens = map(lambda x:st.stem(x),sentenceTokens)
            # if stopwords are required, do the following
            if stopWords:
                sentenceTokens = stripWords(sentenceTokens,stopWords)
            entry = {"sentenceId":sentenceId,"sentence":sentenceTokens,"sentiment":sentiment}
            rawData.append(entry)

    print("Done")
    return rawData

#@timeExec
def createInvertedRawData(text, stopWords = None):
    st = LancasterStemmer()

    invertedRawData = {}
    text.readline()
    print("********** Skipped the first line **********")
    print("********** Creating Inverted Raw Data **********")

    if stopWords:
        signal = 'ON'
    else:
        signal = 'OFF'

    print("Strip Stop-words:" + signal)

    for line in text:
        # Split line
        lineTokens = line.strip('\n').split('\t')

        # get PharseId
        pharseId = int(lineTokens[0])

        # get SentenceId
        sentenceId = int(lineTokens[1])

        # get sentence
        sentenceStr = lineTokens[2]

        # get sentiment
        sentiment = int(lineTokens[3])

        # replace one or more spaces by single space
        # then split
        sentenceTokens = re.sub("\s+"," ",sentenceStr).split(' ')

        sentenceTokens = map(lambda x:x.lower(),sentenceTokens)

        sentenceTokens = map(lambda x:st.stem(x),sentenceTokens)

        #print(sentenceTokens)

        # if stopwords are required, do the following
        if stopWords:
            sentenceTokens = stripWords(sentenceTokens,stopWords)

        # create a sentiment key if doesn't exist
        entry = {"pharseId":pharseId,"sentenceId":sentenceId,"sentence":sentenceTokens}

        if sentiment in invertedRawData:
            # append the sentence to its corresponding sentiment list
            invertedRawData[sentiment].append(entry)
        else:
            invertedRawData[sentiment] = []
            invertedRawData[sentiment].append(entry)
    print("Done")
    return invertedRawData

def stripWords(sentenceTokens,wordsList):

    return filter(lambda x: x not in wordsList, sentenceTokens)

def readArgsFromInput(argv):
    rawDataName = stopWordsName = outputName = directory = None
    # Record the flags that user are using
    usedOpts = []
    try:
        opts, args = getopt.getopt(argv,"hr:s:i1o",["rawdata=","stopwords=","ouput="])
    except getopt.GetoptError:
        print 'run parser.py with flag -h to get help doc'
        sys.exit()
    for opt, arg in opts:
        if opt == '-h':
            print 'Flags List:'
            print '-h Help'
            print '-r <rawdata_filename> Import raw data file'
            print '-s <stopwords_filename> Import stopwords file'
            print "-o <outputfile_filename> Define the name of output file.(It's named as no_stopwords.txt by default)"
            print '-1 Mode 1: Process raw data without stripping the stopwords'
            sys.exit()

        elif opt in ("-r","--rawdata"):
            rawDataName = arg
            usedOpts.append('-r')

        elif opt in ("-s","--stopwords"):
            stopWordsName = arg
            usedOpts.append('-s')

        elif opt in ("-o","--output"):
            outputName = arg
            usedOpts.append('-o')

        elif opt in ("-i"):
            usedOpts.append('-i')
            # TODO:export stats images
            pass
        elif opt in ("-1"):
            usedOpts.append('-1')
            # TODO:Mode 1 -- Process raw data without stripping the stopwords
            pass
        else:
            sys.exit("Invalid Flag(s)")

    # pareser wihout flag -o, then the output will be named as this by default
    if '-o' not in usedOpts:
        outputName = 'default_invertedRawdata.json'

    # cannot proceed without filename of raw data
    if rawDataName == None:
        sys.exit("Cannot proceed wihout the filename of raw data")

    return rawDataName, stopWordsName, outputName, usedOpts

def main(argv):

    rawDataName, stopWordsName, outputName, usedOpts = readArgsFromInput(argv)
    # load raw data
    try:
        rawDataFile = open(rawDataName,'r')

        # parser is in mode 1 then prcoess raw data without stripping the stopwords
        if '-1' in usedOpts:
            invertedRawData = createInvertedRawData(rawDataFile)
            rawDataFile.close()

            rawDataFile = open(rawDataName,'r')
            rawData = extractRawData(rawDataFile)
            rawDataFile.close()

        else:
            stopWordsFile = open(stopWordsName,'r')

            # create stopwords list
            stopWordsList = createStopWordsList(stopWordsFile)

            # create invertedRawData
            invertedRawData = createInvertedRawData(rawDataFile,stopWordsList)

            rawDataFile.close()

            rawDataFile = open(rawDataName,'r')

            # extract rawData from rawDataFile
            rawData = extractRawData(rawDataFile,stopWordsList)
            rawDataFile.close()

            average = stats.getWordAverageSentiment(rawData, 2)

            out.saveWordSentiment(average,"average.txt")

            # Now bag of words is ready for feature construction
            matrix2 = generator.createFeatureBagMatrix(rawData)

        # stats.report(invertedRawData)
        # extract the bigrams from inverted raw data
        bigramsInvertedCollection = bigram.extractBigramsFromInvertedRawData(invertedRawData)
        freqDist = bigram.getFrequencyDist(bigramsInvertedCollection)

        # extract the unigrams from inverted raw data
        unigramsInvertedCollection = unigram.extractUnigramsFromInvertedRawData(invertedRawData)
        unifreqDist = bigram.getFrequencyDist(unigramsInvertedCollection)

        # extract the bigrams from raw data
        bigramsRawData = bigram.extractBigramsFromRawData(rawData)

        # bigram sentences by n

        splitedBigramRawData = splitter.splitSentence(3,bigramsRawData)

        matrix = generator.createFreqMatrix(3,splitedBigramRawData,freqDist)

        # dump all the data
        #jsonInvertedRawData = json.dumps(invertedRawData)
        #jsonBigrams = json.dumps(bigramsInvertedCollection)
        #jsonUnigrams = json.dumps(unigramsInvertedCollection)
        #jsonRawData = json.dumps(rawData)
        #jsonSplitedBigramRawData = json.dumps(splitedBigramRawData)

        rawDataFile.close()
        stopWordsFile.close()

        # save files
        #writeFile(outputName,jsonInvertedRawData)
        #writeFile('rawdata.json',jsonRawData)
        #writeFile('SplitedBigramRawData.json',jsonSplitedBigramRawData)
        #writeFile('bigramsCollection.json',jsonBigrams)
        #writeFile('unigramsCollection.json',jsonUnigrams)
    except IOError as e:
        print "Cannot read raw data file or stop-word file"
if __name__ == "__main__":
    main(sys.argv[1:])
<|endoftext|>


<commit_before>import urllib2, sys, subprocess, os, datetime

date_format = datetime.datetime.now().strftime("%b").upper() + "_" + str(datetime.datetime.now().year % 100)

def download_and_convert_file(download_url):
    response = urllib2.urlopen(download_url)
    file = open("{fn}.pdf".format(fn = date_format), 'wb')
    file.write(response.read())
    file.close()

    os.system('gs -sDEVICE=txtwrite -o ./{fn}.txt ./{fn}.pdf 1> /dev/null'.format(fn = date_format))

    print("Completed")

def parse_text():

def main():
    	download_and_convert_file("http://www.rpi.edu/dept/public_safety/blotter/{fn}.pdf".format(fn = date_format))

if __name__ == "__main__":
    main()<commit_msg>Delete parser.py<commit_after><|endoftext|>


<commit_before>#!/usr/bin/env python3

import pycouchdb
import requests
import sys
import xml.etree.ElementTree as ET

def usage():
    print("usage: ./parser.py <NETXML-FILE>")

class Mantis:
    __db = None

    def __init__(self, *args, **kwargs):
        """Constructor for the Mantis class.

        possible arguments are:
        * dbname: name of database to use, default is "wifinetworks"
        * sourcefile: path to netxml source file
        """
        # assume default config for couchdb
        server = pycouchdb.Server()
        try:
            server.info()
        except requests.exceptions.ConnectionError:
            sys.stderr.write("connecting to server failed\n")
            sys.exit(1)

        # assume databasename, if database does not exists, create it
        if 'dbname' in kwargs.keys():
            dbname = kwargs['dbname']
        else:
            dbname = "wifinetworks"

        try:
            db = server.database(dbname)
        except pycouchdb.exceptions.NotFound:
            server.create(dbname)
        self.__db = server.database(dbname)

        if 'sourcefile' in kwargs.keys():
            filename = kwargs['sourcefile']
            self.parse_xml(filename)

    def parse_xml(self,netxmlfile):
        """Parse given netxml file."""
        updatecounter = 0
        tree = ET.ElementTree()
        try:
            tree = ET.parse(netxmlfile)
        except FileNotFoundError:
            sys.stderr.write("opening file failed\n")
        root = tree.getroot()
        for network in tree.findall('wireless-network'):
            networkdata = {}
            # only handle fixed networks, ignore probes etc.
            if "infrastructure" == network.attrib['type']:
                # extract the data
                ssids = network.findall('SSID')
                ssiddata = []
                for ssidnode in ssids:
                    ssiddata.append( self.extract_ssid_data(ssidnode) )
                    networkdata['ssid'] = ssiddata
                    networkdata['bssid'] = network.find('BSSID').text
                    networkdata['snr-info'] = self.extract_snr_info( network.find('snr-info') )
                    networkdata['gps-info'] = self.extract_gps_info( network.find('gps-info') )
                    # push it into couchdb
                doc = self.__db.save(networkdata)
                updatecounter += 1
        return updatecounter

    def extract_ssid_data(self,rawdata):
        """Extract relevant SSID data from given XML-node.

        This data contains per SSID: maximum data rate, encryption modes and
        ESSID.
        """

        if not isinstance(rawdata,ET.Element):
            raise TypeError("given rawdata not an ElementTree-element")

        ssiddata = {}
        maxrate = rawdata.find('max-rate').text
        ssiddata['max-rate'] = float(maxrate)

        encryption_modes = rawdata.findall('encryption')
        enctxt = []
        for enc in encryption_modes:
            enctxt.append(enc.text)
            ssiddata['encryption'] = enctxt

        ssiddata['essid'] = rawdata.find('essid').text

        return ssiddata

    def extract_snr_info(self,rawdata):
        """Extract relevant radio signal data from given XML-node.

        This data contains minimum and maximum levels of the signal and noise.
        """
        if not isinstance(rawdata,ET.Element):
            raise TypeError("given rawdata not an ElementTree-element")

        snrinfo = {}
        data = rawdata.find('min_signal_dbm').text
        snrinfo['min_signal_dbm'] = int(data)
        data = rawdata.find('min_noise_dbm').text
        snrinfo['min_noise_dbm'] = int(data)
        data = rawdata.find('min_signal_rssi').text
        snrinfo['min_signal_rssi'] = int(data)
        data = rawdata.find('min_noise_rssi').text
        snrinfo['min_noise_rssi'] = int(data)

        data = rawdata.find('max_signal_dbm').text
        snrinfo['max_signal_dbm'] = int(data)
        data = rawdata.find('max_noise_dbm').text
        snrinfo['max_noise_dbm'] = int(data)
        data = rawdata.find('max_signal_rssi').text
        snrinfo['max_signal_rssi'] = int(data)
        data = rawdata.find('max_noise_rssi').text
        snrinfo['max_noise_rssi'] = int(data)

        return snrinfo

    def extract_gps_info(self, rawdata):
        """Extract relevant GPS data from given XML-node.

        This data contains the coordinates of the minimum, maximum and peak
        signal level.
        """

        if not isinstance(rawdata,ET.Element):
            raise TypeError("given rawdata not an ElementTree-element")

        gpsinfo = {}
        data = rawdata.find('min-lat').text
        gpsinfo['min-lat'] = float(data)
        data = rawdata.find('min-lon').text
        gpsinfo['min-lon'] = float(data)
        data = rawdata.find('max-lat').text
        gpsinfo['max-lat'] = float(data)
        data = rawdata.find('max-lon').text
        gpsinfo['max-lon'] = float(data)
        data = rawdata.find('peak-lat').text
        gpsinfo['peak-lat'] = float(data)
        data = rawdata.find('peak-lon').text
        gpsinfo['peak-lon'] = float(data)

        return gpsinfo

    def remove_duplicates(self):
        """Remove all duplicate entries from the database. Return number of removed entries."""
        mapfunction = "function(doc) {\
        var bssid, essid, uid;\
        if (doc.ssid && doc.bssid) {\
            uid = [doc.bssid];\
            names = [];\
            for (index in doc.ssid) {\
                names.push(doc.ssid[index]['essid']);\
            }\
            uid.push(names);\
            emit(uid, null);\
        }\
        }"

        allentries = list(self.__db.temporary_query(mapfunction))
        # this algorithm deletes entries with same UIDs
        # the current UID could be insufficient
        knownids = []
        killlist = []
        for item in allentries:
            if item['key'] not in knownids:
                knownids.append(item['key'])
            else:
                killlist.append(item['id'])

        for docid in killlist:
            self.__db.delete(docid)

        return len(killlist)

# only call if executed as script
if __name__ == '__main__':
    if 2 != len(sys.argv):
        usage()
        sys.exit(1)

    #databucket = Mantis(sourcefile=sys.argv[1])
    databucket = Mantis()
    purges = databucket.remove_duplicates()
    print(purges)
<commit_msg>remove_duplicates -> deflate<commit_after>#!/usr/bin/env python3

import pycouchdb
import requests
import sys
import xml.etree.ElementTree as ET

def usage():
    print("usage: ./parser.py <NETXML-FILE>")

class Mantis:
    __db = None

    def __init__(self, *args, **kwargs):
        """Constructor for the Mantis class.

        possible arguments are:
        * dbname: name of database to use, default is "wifinetworks"
        * sourcefile: path to netxml source file
        """
        # assume default config for couchdb
        server = pycouchdb.Server()
        try:
            server.info()
        except requests.exceptions.ConnectionError:
            sys.stderr.write("connecting to server failed\n")
            sys.exit(1)

        # assume databasename, if database does not exists, create it
        if 'dbname' in kwargs.keys():
            dbname = kwargs['dbname']
        else:
            dbname = "wifinetworks"

        try:
            db = server.database(dbname)
        except pycouchdb.exceptions.NotFound:
            server.create(dbname)
        self.__db = server.database(dbname)

        if 'sourcefile' in kwargs.keys():
            filename = kwargs['sourcefile']
            self.parse_xml(filename)

    def parse_xml(self,netxmlfile):
        """Parse given netxml file."""
        updatecounter = 0
        tree = ET.ElementTree()
        try:
            tree = ET.parse(netxmlfile)
        except FileNotFoundError:
            sys.stderr.write("opening file failed\n")
        root = tree.getroot()
        for network in tree.findall('wireless-network'):
            networkdata = {}
            # only handle fixed networks, ignore probes etc.
            if "infrastructure" == network.attrib['type']:
                # extract the data
                ssids = network.findall('SSID')
                ssiddata = []
                for ssidnode in ssids:
                    ssiddata.append( self.extract_ssid_data(ssidnode) )
                    networkdata['ssid'] = ssiddata
                    networkdata['bssid'] = network.find('BSSID').text
                    networkdata['snr-info'] = self.extract_snr_info( network.find('snr-info') )
                    networkdata['gps-info'] = self.extract_gps_info( network.find('gps-info') )
                    # push it into couchdb
                doc = self.__db.save(networkdata)
                updatecounter += 1
        return updatecounter

    def extract_ssid_data(self,rawdata):
        """Extract relevant SSID data from given XML-node.

        This data contains per SSID: maximum data rate, encryption modes and
        ESSID.
        """

        if not isinstance(rawdata,ET.Element):
            raise TypeError("given rawdata not an ElementTree-element")

        ssiddata = {}
        maxrate = rawdata.find('max-rate').text
        ssiddata['max-rate'] = float(maxrate)

        encryption_modes = rawdata.findall('encryption')
        enctxt = []
        for enc in encryption_modes:
            enctxt.append(enc.text)
            ssiddata['encryption'] = enctxt

        ssiddata['essid'] = rawdata.find('essid').text

        return ssiddata

    def extract_snr_info(self,rawdata):
        """Extract relevant radio signal data from given XML-node.

        This data contains minimum and maximum levels of the signal and noise.
        """
        if not isinstance(rawdata,ET.Element):
            raise TypeError("given rawdata not an ElementTree-element")

        snrinfo = {}
        data = rawdata.find('min_signal_dbm').text
        snrinfo['min_signal_dbm'] = int(data)
        data = rawdata.find('min_noise_dbm').text
        snrinfo['min_noise_dbm'] = int(data)
        data = rawdata.find('min_signal_rssi').text
        snrinfo['min_signal_rssi'] = int(data)
        data = rawdata.find('min_noise_rssi').text
        snrinfo['min_noise_rssi'] = int(data)

        data = rawdata.find('max_signal_dbm').text
        snrinfo['max_signal_dbm'] = int(data)
        data = rawdata.find('max_noise_dbm').text
        snrinfo['max_noise_dbm'] = int(data)
        data = rawdata.find('max_signal_rssi').text
        snrinfo['max_signal_rssi'] = int(data)
        data = rawdata.find('max_noise_rssi').text
        snrinfo['max_noise_rssi'] = int(data)

        return snrinfo

    def extract_gps_info(self, rawdata):
        """Extract relevant GPS data from given XML-node.

        This data contains the coordinates of the minimum, maximum and peak
        signal level.
        """

        if not isinstance(rawdata,ET.Element):
            raise TypeError("given rawdata not an ElementTree-element")

        gpsinfo = {}
        data = rawdata.find('min-lat').text
        gpsinfo['min-lat'] = float(data)
        data = rawdata.find('min-lon').text
        gpsinfo['min-lon'] = float(data)
        data = rawdata.find('max-lat').text
        gpsinfo['max-lat'] = float(data)
        data = rawdata.find('max-lon').text
        gpsinfo['max-lon'] = float(data)
        data = rawdata.find('peak-lat').text
        gpsinfo['peak-lat'] = float(data)
        data = rawdata.find('peak-lon').text
        gpsinfo['peak-lon'] = float(data)

        return gpsinfo

    def deflate(self):
        """Remove all duplicate entries and compact database. Return number of removed entries."""
        mapfunction = "function(doc) {\
        var bssid, essid, uid;\
        if (doc.ssid && doc.bssid) {\
            uid = [doc.bssid];\
            names = [];\
            for (index in doc.ssid) {\
                names.push(doc.ssid[index]['essid']);\
            }\
            uid.push(names);\
            emit(uid, null);\
        }\
        }"

        allentries = list(self.__db.temporary_query(mapfunction))
        # this algorithm deletes entries with same UIDs
        # the current UID could be insufficient
        knownids = []
        killlist = []
        for item in allentries:
            if item['key'] not in knownids:
                knownids.append(item['key'])
            else:
                killlist.append(item['id'])

        for docid in killlist:
            self.__db.delete(docid)

        self.__db.compact()

        return len(killlist)

# only call if executed as script
if __name__ == '__main__':
    if 2 != len(sys.argv):
        usage()
        sys.exit(1)

    #databucket = Mantis(sourcefile=sys.argv[1])
    databucket = Mantis()
    purges = databucket.deflate()
    print(purges)
<|endoftext|>


<commit_before><commit_msg>Deleted all of the merge error lines<commit_after><|endoftext|>


<commit_before>from django import template
from django.shortcuts import get_object_or_404
from django.core.urlresolvers import reverse
from issues.models import Proposal, ProposalVote, VoteResult
from meetings.models import Meeting

register = template.Library()

@register.filter
def voted_on(u, proposal_id):
    voted = ProposalVote.objects.filter(
                            proposal_id=proposal_id,
                            user_id=u.id).exists()
    return voted

@register.filter
def vote_percentage(proposal):
    if proposal.votes_pro is None:
        return 'undefined'
    votes = proposal.votes_pro + proposal.votes_con
    print votes ,proposal.community_members
    percentage = (float(votes) / float(proposal.community_members)) * 100.0
    return str(round(percentage, 1))[:1]
    #return round(percentage, 1)

@register.simple_tag(takes_context=True)
def user_votes_on_issue(context):
    issue = context['i']
    user_id = context['user'].id
    proposals = Proposal.objects.open().filter(issue_id=issue.id)
    votes_cnt = ProposalVote.objects.filter(
        proposal__in=proposals,
        user_id=user_id).count()
    # return "{0}/{1}".format(votes_cnt, proposals.count())
    return "{0}/{1}".format(votes_cnt, proposals.count())

@register.filter
def prev_straw_results_link(proposal, meeting_id=None):
    link_args = {
        'community_id': proposal.issue.community_id,
        'pk': proposal.id,
        }
    url = reverse('vote_results_panel', kwargs=link_args)
    if meeting_id:
        cur_meeting = get_object_or_404(Meeting, id=meeting_id)
        prev_res = VoteResult.objects.filter(proposal=proposal,
                            meeting__held_at__lt=cur_meeting.held_at) \
                            .order_by('-meeting__held_at')

        if prev_res.count():
            return '{0}?meeting_id={1}&dir=prev'.format( \
                     url, prev_res[0].meeting_id)
    else:
        # get meeting from last VoteResult object
        try:
            result = VoteResult.objects.filter(proposal=proposal) \
                                       .latest('meeting__held_at')
            return '{0}?meeting_id={1}&dir=prev'.format( \
                     url, result.meeting.id)
        except VoteResult.DoesNotExist:
            pass

    return ''

@register.filter
def next_straw_results_link(proposal, meeting_id):
    link_args = {
        'community_id': proposal.issue.community_id,
        'pk': proposal.id,
        }
    url = reverse('vote_results_panel', kwargs=link_args)
    if meeting_id:
        cur_meeting = get_object_or_404(Meeting, id=meeting_id)
        next_res = VoteResult.objects.filter(proposal=proposal,
                            meeting__held_at__gt=cur_meeting.held_at) \
                            .order_by('meeting__held_at')
        if next_res.count():
            return '{0}?meeting_id={1}&dir=next'.format( \
                     url, next_res[0].meeting_id)
        elif proposal.can_show_straw_votes:
            return '{0}?meeting_id=&dir=next'.format(url)

    else:
        pass
    return ''

@register.filter
def subtract(value, arg):
    if arg is None:
        return value
    return value - arg
<commit_msg>pythonless programmer :)<commit_after>from django import template
from django.shortcuts import get_object_or_404
from django.core.urlresolvers import reverse
from issues.models import Proposal, ProposalVote, VoteResult
from meetings.models import Meeting

register = template.Library()

@register.filter
def voted_on(u, proposal_id):
    voted = ProposalVote.objects.filter(
                            proposal_id=proposal_id,
                            user_id=u.id).exists()
    return voted

@register.filter
def vote_percentage(proposal):
    if proposal.votes_pro is None:
        return 'undefined'
    votes = proposal.votes_pro + proposal.votes_con
    print votes ,proposal.community_members
    percentage = (float(votes) / float(proposal.community_members)) * 100.0
    return round(percentage, 1)

@register.simple_tag(takes_context=True)
def user_votes_on_issue(context):
    issue = context['i']
    user_id = context['user'].id
    proposals = Proposal.objects.open().filter(issue_id=issue.id)
    votes_cnt = ProposalVote.objects.filter(
        proposal__in=proposals,
        user_id=user_id).count()
    # return "{0}/{1}".format(votes_cnt, proposals.count())
    return "{0}/{1}".format(votes_cnt, proposals.count())

@register.filter
def prev_straw_results_link(proposal, meeting_id=None):
    link_args = {
        'community_id': proposal.issue.community_id,
        'pk': proposal.id,
        }
    url = reverse('vote_results_panel', kwargs=link_args)
    if meeting_id:
        cur_meeting = get_object_or_404(Meeting, id=meeting_id)
        prev_res = VoteResult.objects.filter(proposal=proposal,
                            meeting__held_at__lt=cur_meeting.held_at) \
                            .order_by('-meeting__held_at')

        if prev_res.count():
            return '{0}?meeting_id={1}&dir=prev'.format( \
                     url, prev_res[0].meeting_id)
    else:
        # get meeting from last VoteResult object
        try:
            result = VoteResult.objects.filter(proposal=proposal) \
                                       .latest('meeting__held_at')
            return '{0}?meeting_id={1}&dir=prev'.format( \
                     url, result.meeting.id)
        except VoteResult.DoesNotExist:
            pass

    return ''

@register.filter
def next_straw_results_link(proposal, meeting_id):
    link_args = {
        'community_id': proposal.issue.community_id,
        'pk': proposal.id,
        }
    url = reverse('vote_results_panel', kwargs=link_args)
    if meeting_id:
        cur_meeting = get_object_or_404(Meeting, id=meeting_id)
        next_res = VoteResult.objects.filter(proposal=proposal,
                            meeting__held_at__gt=cur_meeting.held_at) \
                            .order_by('meeting__held_at')
        if next_res.count():
            return '{0}?meeting_id={1}&dir=next'.format( \
                     url, next_res[0].meeting_id)
        elif proposal.can_show_straw_votes:
            return '{0}?meeting_id=&dir=next'.format(url)

    else:
        pass
    return ''

@register.filter
def subtract(value, arg):
    if arg is None:
        return value
    return value - arg
<|endoftext|>


<commit_before>import lldb
from lldbsuite.test.lldbtest import *
from lldbsuite.test.decorators import *
from gdbclientutils import *

class TestGDBRemoteLoad(GDBRemoteTestBase):

    def test_ram_load(self):
        """Test loading an object file to a target's ram"""
        target = self.createTarget("a.yaml")
        process = self.connect(target)
        self.dbg.HandleCommand("target modules load -l -s0")
        self.assertPacketLogContains([
                "M1000,4:c3c3c3c3",
                "M1004,2:3232"
                ])

    @skipIfXmlSupportMissing
    def test_flash_load(self):
        """Test loading an object file to a target's flash memory"""

        class Responder(MockGDBServerResponder):
            def qSupported(self, client_supported):
                return "PacketSize=3fff;QStartNoAckMode+;qXfer:memory-map:read+"

            def qXferRead(self, obj, annex, offset, length):
                if obj == "memory-map":
                    return (self.MEMORY_MAP[offset:offset + length],
                            offset + length < len(self.MEMORY_MAP))
                return None, False

            def other(self, packet):
                if packet[0:11] == "vFlashErase":
                    return "OK"
                if packet[0:11] == "vFlashWrite":
                    return "OK"
                if packet == "vFlashDone":
                    return "OK"
                return ""

            MEMORY_MAP = """<?xml version="1.0"?>
<memory-map>
  <memory type="ram" start="0x0" length="0x1000"/>
  <memory type="flash" start="0x1000" length="0x1000">
    <property name="blocksize">0x100</property>
  </memory>
  <memory type="ram" start="0x2000" length="0x1D400"/>
</memory-map>
"""

        self.server.responder = Responder()
        target = self.createTarget("a.yaml")
        process = self.connect(target)
        self.dbg.HandleCommand("target modules load -l -s0")
        self.assertPacketLogContains([
                "vFlashErase:1000,100",
                "vFlashWrite:1000:\xc3\xc3\xc3\xc3",
                "vFlashWrite:1004:\x32\x32",
                "vFlashDone"
                ])
<commit_msg>Add a test for setting the load address of a module with differing physical/virtual addresses<commit_after>import lldb
from lldbsuite.test.lldbtest import *
from lldbsuite.test.decorators import *
from gdbclientutils import *

class TestGDBRemoteLoad(GDBRemoteTestBase):

    def setUp(self):
        super(TestGDBRemoteLoad, self).setUp()
        self._initial_platform = lldb.DBG.GetSelectedPlatform()

    def tearDown(self):
        lldb.DBG.SetSelectedPlatform(self._initial_platform)
        super(TestGDBRemoteLoad, self).tearDown()

    def test_module_load_address(self):
        """Test that setting the load address of a module uses virtual addresses"""
        target = self.createTarget("a.yaml")
        process = self.connect(target)
        module = target.GetModuleAtIndex(0)
        self.assertTrue(module.IsValid())
        self.assertTrue(target.SetModuleLoadAddress(module, 0).Success())
        address = target.ResolveLoadAddress(0x2001)
        self.assertTrue(address.IsValid())
        self.assertEqual(".data", address.GetSection().GetName())

    def test_ram_load(self):
        """Test loading an object file to a target's ram"""
        target = self.createTarget("a.yaml")
        process = self.connect(target)
        self.dbg.HandleCommand("target modules load -l -s0")
        self.assertPacketLogContains([
                "M1000,4:c3c3c3c3",
                "M1004,2:3232"
                ])

    @skipIfXmlSupportMissing
    def test_flash_load(self):
        """Test loading an object file to a target's flash memory"""

        class Responder(MockGDBServerResponder):
            def qSupported(self, client_supported):
                return "PacketSize=3fff;QStartNoAckMode+;qXfer:memory-map:read+"

            def qXferRead(self, obj, annex, offset, length):
                if obj == "memory-map":
                    return (self.MEMORY_MAP[offset:offset + length],
                            offset + length < len(self.MEMORY_MAP))
                return None, False

            def other(self, packet):
                if packet[0:11] == "vFlashErase":
                    return "OK"
                if packet[0:11] == "vFlashWrite":
                    return "OK"
                if packet == "vFlashDone":
                    return "OK"
                return ""

            MEMORY_MAP = """<?xml version="1.0"?>
<memory-map>
  <memory type="ram" start="0x0" length="0x1000"/>
  <memory type="flash" start="0x1000" length="0x1000">
    <property name="blocksize">0x100</property>
  </memory>
  <memory type="ram" start="0x2000" length="0x1D400"/>
</memory-map>
"""

        self.server.responder = Responder()
        target = self.createTarget("a.yaml")
        process = self.connect(target)
        self.dbg.HandleCommand("target modules load -l -s0")
        self.assertPacketLogContains([
                "vFlashErase:1000,100",
                "vFlashWrite:1000:\xc3\xc3\xc3\xc3",
                "vFlashWrite:1004:\x32\x32",
                "vFlashDone"
                ])
<|endoftext|>