<commit_before>import os

from ctypes import c_char, c_char_p, c_ubyte, c_void_p, c_int
from ctypes import c_uint, c_uint8, c_uint32, c_uint64
from ctypes import CDLL, POINTER, Structure, Union, byref, cast, pointer
from itertools import product
from operator import __and__, __or__

from .traildb import tdb, tdb_field, tdb_item, tdb_fold_fn

cd = os.path.dirname(os.path.abspath(__file__))
lib = CDLL(os.path.join(cd, 'libtraildb.so'))

def api(fun, args, res=None):
    fun.argtypes = args
    fun.restype = res

def struct(fields):
    class struct(Structure):
        _fields_ = fields
    return struct

def union(fields):
    class union(Union):
        _fields_ = fields
    return union

FDB_DENSE   = 1
FDB_SPARSE  = 2

FDB_SIMPLE  = 1
FDB_COMPLEX = 2

fdb_cons = c_void_p
fdb_eid  = c_uint32
fdb_fid  = c_uint32
fdb_mask = c_uint8

FDB = struct([('data_offs', c_uint64),
              ('data_size', c_uint64),
              ('params', c_uint8 * 1024),
              ('num_funnels', fdb_fid)])
fdb = POINTER(FDB)

FDB_EZ = struct([('num_keys', c_uint),
                 ('key_offs', fdb_fid * 128),
                 ('key_fields', tdb_field * 128),
                 ('mask_field', tdb_field)])
fdb_ez = POINTER(FDB_EZ)

FDB_ELEM = struct([('id', fdb_eid),
                   ('mask', fdb_mask)])
fdb_elem = POINTER(FDB_ELEM)

FDB_CLAUSE = struct([('terms', c_uint64),
                     ('nterms', c_uint64)])
fdb_clause = POINTER(FDB_CLAUSE)

FDB_CNF = struct([('num_clauses', c_uint),
                  ('clauses', fdb_clause)])
fdb_cnf = POINTER(FDB_CNF)

class FDB_SET(Structure):
    pass
fdb_set = POINTER(FDB_SET)

FDB_SET_SIMPLE = struct([('db', fdb),
                         ('funnel_id', fdb_fid),
                         ('cnf', fdb_cnf)])
fdb_set_simple = POINTER(FDB_SET_SIMPLE)

FDB_SET_COMPLEX = struct([('db', fdb),
                          ('num_sets', c_uint),
                          ('sets', fdb_set),
                          ('cnf', fdb_cnf)])
fdb_set_complex = POINTER(FDB_SET_SIMPLE)

FDB_SET._anonymous_ = ['_impl']
FDB_SET._fields_ = [('flags', c_uint8),
                    ('_impl', union([('simple', FDB_SET_SIMPLE),
                                     ('complex', FDB_SET_COMPLEX)]))]

FDB_FAMILY = struct([('db', fdb),
                     ('num_sets', c_uint),
                     ('funnel_id', fdb_fid),
                     ('cnfs', fdb_cnf)])
fdb_family = POINTER(FDB_FAMILY)

fdb_iter = c_void_p

api(lib.fdb_create, [tdb, tdb_fold_fn, fdb_fid, c_void_p], fdb)
api(lib.fdb_detect, [fdb_fid, fdb_eid, fdb_mask, fdb_cons])

api(lib.fdb_easy, [tdb, fdb_ez], fdb)
api(lib.fdb_dump, [fdb, c_int], fdb)
api(lib.fdb_load, [c_int], fdb)
api(lib.fdb_free, [fdb], fdb)

api(lib.fdb_iter_new, [fdb_set], fdb_iter);
api(lib.fdb_iter_next, [fdb_iter], fdb_elem);
api(lib.fdb_iter_free, [fdb_iter], fdb_iter);

api(lib.fdb_count_set, [fdb_set, POINTER(fdb_eid)], c_int)
api(lib.fdb_count_family, [fdb_family, POINTER(fdb_eid)], c_int)

def keys(num_keys, key_fields):
    key = []
    for i, f in enumerate(key_fields):
        if not num_keys:
            return
        if f:
            key.append(f)
        else:
            num_keys -= 1
            yield tuple(sorted(key)), i - len(key)
            key = []

def seqify(x):
    try:
        x[0]
        return x
    except IndexError:
        return x
    except TypeError:
        return x,

class FunnelDBError(Exception):
    pass

class FunnelDB(object):
    def __init__(self, traildb, db):
        self.db = db
        self.traildb = traildb
        self.num_funnels = db.contents.num_funnels
        self.params = cast(db.contents.params, fdb_ez).contents
        self.keydict = dict(keys(self.params.num_keys, self.params.key_fields))
        self.mask = traildb.fields[self.params.mask_field]

    def __del__(self):
        lib.fdb_free(self.db)

    def __len__(self):
        return self.num_funnels

    def __getitem__(self, id):
        return self.funnel(id)

    def funnel(self, id, cnf=None):
        if id < 0 or id >= len(self):
            raise IndexError("Invalid funnel id: %s" % id)
        items = FDB_SET_SIMPLE(db=self.db, funnel_id=id, cnf=cnf)
        index = fdb_eid()
        next_ = FDB_ELEM()
        while True:
            if not lib.fdb_next(items, byref(index), byref(next_)):
                return
            yield next_.id, next_.mask

    def key(self, **which):
        tdb = self.traildb
        O, F = self.params.key_offs, self.params.key_fields
        which = dict((tdb.fields.index(k), v) for k, v in which.items())
        index = self.keydict[tuple(sorted(which))]
        key = O[index - 1] if index else 0
        for i, f in enumerate(F[index:]):
            if not f:
                break
            key += tdb.val(f, which[f]) * O[index + i]
        return key

    def keys(self):
        tdb = self.traildb
        for _, key in sorted((v, k) for k, v in self.keydict.items()):
            names = [tdb.fields[f] for f in key]
            for which in product(*(tdb.lexicon(f) for f in key)):
                yield dict(zip(names, which))

    def path_key(self, path):
        return self.key(**dict(i.split(':') for i in path.split('/') if i))

    def simple_cnfs(self, mask_filters):
        if isinstance(mask_filters[0], FDB_CNF):
            return mask_filters
        f = self.params.mask_field
        tdb = self.traildb
        cnfs = (FDB_CNF * len(mask_filters))()
        for i, mask_filter in enumerate(mask_filters):
            if mask_filter and mask_filter != '*':
                q = Q.parse(mask_filter, ext=True)
                cnfs[i].num_clauses = len(q.clauses)
                cnfs[i].clauses = (FDB_CLAUSE * len(q.clauses))()
                for n, c in enumerate(q.clauses):
                    for l in c.literals:
                        if l.negated:
                            cnfs[i].clauses[n].nterms |= 1 << tdb.val(f, l.term)
                        else:
                            cnfs[i].clauses[n].terms |= 1 << tdb.val(f, l.term)
            else:
                cnfs[i].num_clauses = 0
        return cnfs

    def simple_sets(self, terms):
        sets = (FDB_SET * len(terms))()
        for i, term in enumerate(terms):
            path, mask_filter = term.split('=')
            sets[i].flags = FDB_SIMPLE
            sets[i].simple.db = self.db
            sets[i].simple.funnel_id = self.path_key(path)
            sets[i].simple.cnf = self.simple_cnfs([mask_filter])
        return sets

    def complex_set(self, query):
        tdb = self.traildb
        q = Q.parse(query)
        terms = list(set(l.term for c in q.clauses for l in c.literals))
        cset = FDB_SET()
        cnf = FDB_CNF()
        cset.flags = FDB_COMPLEX
        cset.complex.db = self.db
        cset.complex.num_sets = len(terms)
        cset.complex.sets = self.simple_sets(terms)
        cset.complex.cnf = pointer(cnf)
        cnf.num_clauses = len(q.clauses)
        cnf.clauses = (FDB_CLAUSE * len(q.clauses))()
        for n, c in enumerate(q.clauses):
            for l in c.literals:
                if l.negated:
                    cnf.clauses[n].nterms |= 1 << terms.index(l.term)
                else:
                    cnf.clauses[n].terms |= 1 << terms.index(l.term)
        return pointer(cset)

    def count_set(self, query):
        count = fdb_eid()
        lib.fdb_count_set(self.complex_set(query), pointer(count))
        return count.value

    def count_families(self, keys, mask_filters=None):
        fmasks = seqify(mask_filters)
        family = FDB_FAMILY(db=self.db)
        family.num_sets = len(fmasks)
        family.cnfs = self.simple_cnfs(fmasks)
        counts = (fdb_eid * family.num_sets)()
        for key in seqify(keys):
            family.funnel_id = key
            lib.fdb_count_family(family, counts)
            yield key, zip(fmasks, counts)

    @classmethod
    def easy(cls, traildb, keys=((),), mask_field=1):
        params = FDB_EZ(num_keys=len(keys), mask_field=mask_field)
        i = 0
        for group in keys:
            for k in seqify(group):
                params.key_fields[i] = traildb.fields.index(k)
                i += 1
            i += 1
        return cls(traildb, lib.fdb_easy(traildb._db, params))

    @classmethod
    def load(cls, traildb, path):
        with open(path) as file:
            db = lib.fdb_load(file.fileno())
        if not db:
            raise FunnelDBError("Could not open %s" % path)
        return cls(traildb, db)

    def dump(self, path):
        with open(path, 'w') as file:
            lib.fdb_dump(self.db, file.fileno())

class Q(object):
    def __init__(self, clauses):
        self.clauses = frozenset(clauses)

    def __and__(self, other):
        return Q(self.clauses | other.clauses)

    def __or__(self, other):
        return Q(c | d for c in self.clauses for d in other.clauses)

    def __invert__(self):
        if not self.clauses:
            return self
        return Q.wrap(reduce(__or__, (~c for c in self.clauses)))

    def __cmp__(self, other):
        return cmp(str(self), str(other))

    def __eq__(self, other):
        return self.clauses == other.clauses

    def __hash__(self):
        return hash(self.clauses)

    def __str__(self):
        format = '(%s)' if len(self.clauses) > 1 else '%s'
        return ','.join(format % c for c in self.clauses)

    @classmethod
    def parse(cls, string, ext=False):
        import re
        if ext:
            string = string.replace('!', '~').replace(',', '&').replace('+', '|')
        return eval(re.sub(r'([^&|~()\s][^&|~()]*)',
                           r'Q.wrap("""\1""".strip())', string) or 'Q([])')

    @classmethod
    def wrap(cls, proposition):
        if isinstance(proposition, Q):
            return proposition
        if isinstance(proposition, Clause):
            return Q((proposition, ))
        if isinstance(proposition, Literal):
            return Q((Clause((proposition, )), ))
        return Q((Clause((Literal(proposition), )), ))

class Clause(object):
    def __init__(self, literals):
        self.literals = frozenset(literals)

    def __and__(self, other):
        return Q((self, other))

    def __or__(self, other):
        return Clause(self.literals | other.literals)

    def __invert__(self):
        return Q.wrap(reduce(__and__, (~l for l in self.literals)))

    def __eq__(self, other):
        return self.literals == other.literals

    def __hash__(self):
        return hash(self.literals)

    def __str__(self):
        return '+'.join('%s' % l for l in self.literals)

class Literal(object):
    def __init__(self, term, negated=False):
        self.term    = term
        self.negated = negated

    def __and__(self, other):
        return Clause((self, )) & Clause((other, ))

    def __or__(self, other):
        return Clause((self, other))

    def __invert__(self):
        return type(self)(self.term, negated=not self.negated)

    def __eq__(self, other):
        return self.term == other.term and self.negated == other.negated

    def __hash__(self):
        return hash(self.term) ^ hash(self.negated)

    def __str__(self):
        return '%s%s' % ('!' if self.negated else '', self.term)
<commit_msg>some helpers for queries with lots of terms; Q bugfixes<commit_after>import os

from ctypes import c_char, c_char_p, c_ubyte, c_void_p, c_int
from ctypes import c_uint, c_uint8, c_uint32, c_uint64
from ctypes import CDLL, POINTER, Structure, Union, byref, cast, pointer
from itertools import product
from operator import __and__, __or__

from .traildb import tdb, tdb_field, tdb_item, tdb_fold_fn

cd = os.path.dirname(os.path.abspath(__file__))
lib = CDLL(os.path.join(cd, 'libtraildb.so'))

def api(fun, args, res=None):
    fun.argtypes = args
    fun.restype = res

def struct(fields):
    class struct(Structure):
        _fields_ = fields
    return struct

def union(fields):
    class union(Union):
        _fields_ = fields
    return union

FDB_DENSE   = 1
FDB_SPARSE  = 2

FDB_SIMPLE  = 1
FDB_COMPLEX = 2

fdb_cons = c_void_p
fdb_eid  = c_uint32
fdb_fid  = c_uint32
fdb_mask = c_uint8

FDB = struct([('data_offs', c_uint64),
              ('data_size', c_uint64),
              ('params', c_uint8 * 1024),
              ('num_funnels', fdb_fid)])
fdb = POINTER(FDB)

FDB_EZ = struct([('num_keys', c_uint),
                 ('key_offs', fdb_fid * 128),
                 ('key_fields', tdb_field * 128),
                 ('mask_field', tdb_field)])
fdb_ez = POINTER(FDB_EZ)

FDB_ELEM = struct([('id', fdb_eid),
                   ('mask', fdb_mask)])
fdb_elem = POINTER(FDB_ELEM)

FDB_CLAUSE = struct([('terms', c_uint64),
                     ('nterms', c_uint64)])
fdb_clause = POINTER(FDB_CLAUSE)

FDB_CNF = struct([('num_clauses', c_uint),
                  ('clauses', fdb_clause)])
fdb_cnf = POINTER(FDB_CNF)

class FDB_SET(Structure):
    pass
fdb_set = POINTER(FDB_SET)

FDB_SET_SIMPLE = struct([('db', fdb),
                         ('funnel_id', fdb_fid),
                         ('cnf', fdb_cnf)])
fdb_set_simple = POINTER(FDB_SET_SIMPLE)

FDB_SET_COMPLEX = struct([('db', fdb),
                          ('num_sets', c_uint),
                          ('sets', fdb_set),
                          ('cnf', fdb_cnf)])
fdb_set_complex = POINTER(FDB_SET_SIMPLE)

FDB_SET._anonymous_ = ['_impl']
FDB_SET._fields_ = [('flags', c_uint8),
                    ('_impl', union([('simple', FDB_SET_SIMPLE),
                                     ('complex', FDB_SET_COMPLEX)]))]

FDB_FAMILY = struct([('db', fdb),
                     ('num_sets', c_uint),
                     ('funnel_id', fdb_fid),
                     ('cnfs', fdb_cnf)])
fdb_family = POINTER(FDB_FAMILY)

fdb_iter = c_void_p

api(lib.fdb_create, [tdb, tdb_fold_fn, fdb_fid, c_void_p], fdb)
api(lib.fdb_detect, [fdb_fid, fdb_eid, fdb_mask, fdb_cons])

api(lib.fdb_easy, [tdb, fdb_ez], fdb)
api(lib.fdb_dump, [fdb, c_int], fdb)
api(lib.fdb_load, [c_int], fdb)
api(lib.fdb_free, [fdb], fdb)

api(lib.fdb_iter_new, [fdb_set], fdb_iter);
api(lib.fdb_iter_next, [fdb_iter], fdb_elem);
api(lib.fdb_iter_free, [fdb_iter], fdb_iter);

api(lib.fdb_count_set, [fdb_set, POINTER(fdb_eid)], c_int)
api(lib.fdb_count_family, [fdb_family, POINTER(fdb_eid)], c_int)

def keys(num_keys, key_fields):
    key = []
    for i, f in enumerate(key_fields):
        if not num_keys:
            return
        if f:
            key.append(f)
        else:
            num_keys -= 1
            yield tuple(sorted(key)), i - len(key)
            key = []

def seqify(x):
    try:
        x[0]
        return x
    except IndexError:
        return x
    except TypeError:
        return x,

def split(terms, n):
    buckets = [[] for i in xrange(n)]
    for i, term in enumerate(terms):
        buckets[i % n].append(term)
    return buckets

class FunnelDBError(Exception):
    pass

class FunnelDB(object):
    def __init__(self, traildb, db):
        self.db = db
        self.traildb = traildb
        self.num_funnels = db.contents.num_funnels
        self.params = cast(db.contents.params, fdb_ez).contents
        self.keydict = dict(keys(self.params.num_keys, self.params.key_fields))
        self.mask = traildb.fields[self.params.mask_field]

    def __del__(self):
        lib.fdb_free(self.db)

    def __len__(self):
        return self.num_funnels

    def __getitem__(self, id):
        return self.funnel(id)

    def funnel(self, id, cnf=None):
        if id < 0 or id >= len(self):
            raise IndexError("Invalid funnel id: %s" % id)
        items = FDB_SET_SIMPLE(db=self.db, funnel_id=id, cnf=cnf)
        index = fdb_eid()
        next_ = FDB_ELEM()
        while True:
            if not lib.fdb_next(items, byref(index), byref(next_)):
                return
            yield next_.id, next_.mask

    def key(self, **which):
        tdb = self.traildb
        O, F = self.params.key_offs, self.params.key_fields
        which = dict((tdb.fields.index(k), v) for k, v in which.items())
        index = self.keydict[tuple(sorted(which))]
        key = O[index - 1] if index else 0
        for i, f in enumerate(F[index:]):
            if not f:
                break
            key += tdb.val(f, which[f]) * O[index + i]
        return key

    def keys(self):
        tdb = self.traildb
        for _, key in sorted((v, k) for k, v in self.keydict.items()):
            names = [tdb.fields[f] for f in key]
            for which in product(*(tdb.lexicon(f) for f in key)):
                yield dict(zip(names, which))

    def path_key(self, path):
        return self.key(**dict(i.split(':') for i in path.split('/') if i))

    def simple_cnfs(self, mask_filters):
        if isinstance(mask_filters[0], FDB_CNF):
            return mask_filters
        f = self.params.mask_field
        tdb = self.traildb
        cnfs = (FDB_CNF * len(mask_filters))()
        for i, mask_filter in enumerate(mask_filters):
            if mask_filter and mask_filter != '*':
                q = Q.parse(mask_filter, ext=True)
                cnfs[i].num_clauses = len(q.clauses)
                cnfs[i].clauses = (FDB_CLAUSE * len(q.clauses))()
                for n, c in enumerate(q.clauses):
                    for l in c.literals:
                        if l.negated:
                            cnfs[i].clauses[n].nterms |= 1 << tdb.val(f, l.term)
                        else:
                            cnfs[i].clauses[n].terms |= 1 << tdb.val(f, l.term)
            else:
                cnfs[i].num_clauses = 0
        return cnfs

    def fdb_sets(self, terms, sets=None):
        sets = sets or (FDB_SET * len(terms))()
        for i, term in enumerate(terms):
            if isinstance(term, FDB_SET):
                sets[i] = term
            else:
                if isinstance(term, basestring):
                    path, mask_filter = term.split('=')
                    funnel_id = self.path_key(path)
                else:
                    funnel_id, mask_filter = term
                sets[i].flags = FDB_SIMPLE
                sets[i].simple.db = self.db
                sets[i].simple.funnel_id = funnel_id
                sets[i].simple.cnf = self.simple_cnfs([mask_filter])
        return sets

    def complex_set(self, query):
        q = Q.parse(query)
        terms = list(set(l.term for c in q.clauses for l in c.literals))
        cset = FDB_SET()
        cnf = FDB_CNF()
        cset.flags = FDB_COMPLEX
        cset.complex.db = self.db
        cset.complex.num_sets = len(terms)
        cset.complex.sets = self.fdb_sets(terms)
        cset.complex.cnf = pointer(cnf)
        cnf.num_clauses = len(q.clauses)
        cnf.clauses = (FDB_CLAUSE * len(q.clauses))()
        for n, c in enumerate(q.clauses):
            for l in c.literals:
                if l.negated:
                    cnf.clauses[n].nterms |= 1 << terms.index(l.term)
                else:
                    cnf.clauses[n].terms |= 1 << terms.index(l.term)
        return pointer(cset)

    def count_set(self, set):
        count = fdb_eid()
        if isinstance(set, FDB_SET):
            lib.fdb_count_set(set, pointer(count))
        else:
            lib.fdb_count_set(self.complex_set(set), pointer(count))
        return count.value

    def count_families(self, keys, mask_filters=None):
        fmasks = seqify(mask_filters)
        family = FDB_FAMILY(db=self.db)
        family.num_sets = len(fmasks)
        family.cnfs = self.simple_cnfs(fmasks)
        counts = (fdb_eid * family.num_sets)()
        for key in seqify(keys):
            family.funnel_id = key
            lib.fdb_count_family(family, counts)
            yield key, zip(fmasks, counts)

    def conjunction(self, terms, cset=None, N=64):
        cset = cset or FDB_SET()
        cset.flags = FDB_COMPLEX
        cset.complex.db = self.db
        cnf = FDB_CNF()
        cnf.num_clauses = N
        cnf.clauses = (FDB_CLAUSE * N)()
        for i in xrange(N):
            cnf.clauses[i].terms = 1 << i
        cset.complex.cnf = pointer(cnf)
        if len(terms) <= N:
            cset.complex.num_sets = len(terms)
            cset.complex.sets = self.fdb_sets(terms)
        else:
            cset.complex.num_sets = N
            cset.complex.sets = (FDB_SET * N)()
            for i, bucket in enumerate(split(terms, n=N)):
                self.conjunction(bucket, cset.complex.sets[i])
        return cset

    def disjunction(self, terms, cset=None, N=64):
        cset = cset or FDB_SET()
        cset.flags = FDB_COMPLEX
        cset.complex.db = self.db
        cnf = FDB_CNF()
        cnf.num_clauses = 1
        cnf.clauses = (FDB_CLAUSE * 1)()
        cnf.clauses[0].terms = (1 << N) - 1
        cset.complex.cnf = pointer(cnf)
        if len(terms) <= N:
            cset.complex.num_sets = len(terms)
            cset.complex.sets = self.fdb_sets(terms)
        else:
            cset.complex.num_sets = N
            cset.complex.sets = (FDB_SET * N)()
            for i, bucket in enumerate(split(terms, n=N)):
                self.disjunction(bucket, cset.complex.sets[i])
        return cset

    def difference(self, include, exclude):
        cset = FDB_SET()
        cset.flags = FDB_COMPLEX
        cset.complex.db = self.db
        cset.complex.num_sets = 2
        cset.complex.sets = (FDB_SET * 2)()
        cset.complex.sets[0] = include
        cset.complex.sets[1] = exclude
        cnf = FDB_CNF()
        cnf.num_clauses = 2
        cnf.clauses = (FDB_CLAUSE * 2)()
        cnf.clauses[0].terms = 1
        cnf.clauses[1].nterms = 2
        cset.complex.cnf = pointer(cnf)
        return cset

    @classmethod
    def easy(cls, traildb, keys=((),), mask_field=1):
        params = FDB_EZ(num_keys=len(keys), mask_field=mask_field)
        i = 0
        for group in keys:
            for k in seqify(group):
                params.key_fields[i] = traildb.fields.index(k)
                i += 1
            i += 1
        return cls(traildb, lib.fdb_easy(traildb._db, params))

    @classmethod
    def load(cls, traildb, path):
        with open(path) as file:
            db = lib.fdb_load(file.fileno())
        if not db:
            raise FunnelDBError("Could not open %s" % path)
        return cls(traildb, db)

    def dump(self, path):
        with open(path, 'w') as file:
            lib.fdb_dump(self.db, file.fileno())

class Q(object):
    def __init__(self, clauses):
        self.clauses = frozenset(c for c in clauses if c.literals)

    def __and__(self, other):
        return Q(self.clauses | other.clauses)

    def __or__(self, other):
        if not self.clauses:
            return other
        if not other.clauses:
            return self
        return Q(c | d for c in self.clauses for d in other.clauses)

    def __invert__(self):
        if not self.clauses:
            return self
        return Q.wrap(reduce(__or__, (~c for c in self.clauses)))

    def __cmp__(self, other):
        return cmp(str(self), str(other))

    def __eq__(self, other):
        return self.clauses == other.clauses

    def __hash__(self):
        return hash(self.clauses)

    def __str__(self):
        format = '(%s)' if len(self.clauses) > 1 else '%s'
        return ','.join(format % c for c in self.clauses)

    @classmethod
    def parse(cls, string, ext=False):
        import re
        if isinstance(string, cls):
            return string
        if ext:
            string = string.replace('!', '~').replace(',', '&').replace('+', '|')
        return eval(re.sub(r'([^&|~()\s][^&|~()]*)',
                           r'Q.wrap("""\1""".strip())', string) or 'Q([])')

    @classmethod
    def wrap(cls, proposition):
        if isinstance(proposition, Q):
            return proposition
        if isinstance(proposition, Clause):
            return Q((proposition, ))
        if isinstance(proposition, Literal):
            return Q((Clause((proposition, )), ))
        return Q((Clause((Literal(proposition), )), ))

class Clause(object):
    def __init__(self, literals):
        self.literals = frozenset(literals)

    def __and__(self, other):
        return Q.wrap(self) & Q.wrap(other)

    def __or__(self, other):
        return Clause(self.literals | other.literals)

    def __invert__(self):
        return Q(Clause([~l]) for l in self.literals)

    def __eq__(self, other):
        return self.literals == other.literals

    def __hash__(self):
        return hash(self.literals)

    def __str__(self):
        return '+'.join('%s' % l for l in self.literals)

class Literal(object):
    def __init__(self, term, negated=False):
        self.term    = term
        self.negated = negated

    def __and__(self, other):
        return Clause((self, )) & Clause((other, ))

    def __or__(self, other):
        return Clause((self, other))

    def __invert__(self):
        return type(self)(self.term, negated=not self.negated)

    def __eq__(self, other):
        return self.term == other.term and self.negated == other.negated

    def __hash__(self):
        return hash(self.term) ^ hash(self.negated)

    def __str__(self):
        return '%s%s' % ('!' if self.negated else '', self.term)
<|endoftext|>


<commit_before>import doctest
import os
import unittest
import matplotlib

matplotlib.use('Agg')

def load_tests(loader, tests, ignore):
    for root, dirs, files in os.walk("./_labsheets"):
        for f in files:
            if f.endswith(".md"):
                 tests.addTests(doctest.DocFileSuite(os.path.join(root, f),
                                                  optionflags=doctest.ELLIPSIS))

    return tests

def clean_up():
    try:
        os.remove("my_plot.pdf")
    except FileNotFoundError:
        pass

def main():
    unittest.main()
    clean_up()

if __name__ == '__main__':
    main()
<commit_msg>Add doctest for handouts.<commit_after>import doctest
import os
import unittest
import matplotlib

matplotlib.use('Agg')

directories = ["./_labsheets", "./_handouts"]

def load_tests(loader, tests, ignore):
    for directory in directories:
        for root, dirs, files in os.walk(directory):
            for f in files:
                if f.endswith(".md"):
                     tests.addTests(doctest.DocFileSuite(os.path.join(root, f),
                                                      optionflags=doctest.ELLIPSIS))

    return tests

def clean_up():
    try:
        os.remove("my_plot.pdf")
    except FileNotFoundError:
        pass

def main():
    unittest.main()
    clean_up()

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>"""This small script offers a python equivalent to "which"

For every argument on the command line it lists
    all python files and all directories with that name
    in each of the directories in sys.path

If a name is found more than once each is listed
    (but python will only import one, probably the first listed)
"""

_copyright = """
(c) J Alan Brogan 2013
    The source is released under the MIT license
    See http://jalanb.mit-license.org/ for more information
"""

import os
import sys
import fnmatch

def directory_list(path):
    """A list of all items in that path

    If path is not a directory, an empty list
    """
    if not os.path.isdir(path):
        return []
    return os.listdir(path)

def is_file_in(path, name):
    """Whether that name is a file in the directory at that path"""
    path_to_name = os.path.join(path, name)
    return os.path.isfile(path_to_name)

def is_matching_file_in(path, name, glob):
    """Whether that name is a file in path and matches that glob"""
    return is_file_in(path, name) and fnmatch.fnmatch(name, glob)

def path_to_module(path, name):
    """Whether the name matches a python source or compiled file in that path

    If source and compiled files are found, give the source
    """
    glob = '%s.py*' % name
    python_files = [f for f in directory_list(path) if is_matching_file_in(path, f, glob)]
    if not python_files:
        return None
    source_files = [f for f in python_files if os.path.splitext(f)[-1] == '.py']
    python_file = source_files and source_files[0] or python_files[0]
    return os.path.join(path, python_file)

def path_to_sub_directory(path, name):
    """If name is a real sub-directory of path, return that"""
    result = os.path.join(path, name)
    if os.path.isdir(result):
        return result

def path_to_python(path, name):
    """Path to either a module or sub-dir of that path, with that name"""
    result = path_to_sub_directory(path, name)
    if result:
        return result
    return path_to_module(path, name)

def built_in(name):
    """Whether the name is that of one of python's builtin modules"""
    try:
        #  Not all builtin modules are initially imported, so bring it in first
        __import__(name)
    except ImportError:
        return False
    return '(built-in)' in str(sys.modules[name])

def main(strings):
    """Run the program"""
    for string in strings:
        if built_in(string):
            print 'builtin', string
        for path in sys.path:
            path_to_string = path_to_python(path, string)
            if path_to_string:
                os.system('ls -ld %s' % path_to_string)

if __name__ == '__main__':
    args = sys.argv[1:]
    if not args:
        from what import test
        sys.exit(test())
    else:
        sys.exit(main(args))
<commit_msg>prefer importlib to hand-crafted shite<commit_after>"""This small script offers a python equivalent to "which"

For every argument on the command line it lists
    all python files and all directories with that name
    in each of the directories in sys.path

If a name is found more than once each is listed
    (but python will only import one, probably the first listed)
"""

_copyright = """
(c) J Alan Brogan 2013
    The source is released under the MIT license
    See http://jalanb.mit-license.org/ for more information
"""

import os
import sys
import fnmatch
import importlib

def directory_list(path):
    """A list of all items in that path

    If path is not a directory, an empty list
    """
    if not os.path.isdir(path):
        return []
    return os.listdir(path)

def is_file_in(path, name):
    """Whether that name is a file in the directory at that path"""
    path_to_name = os.path.join(path, name)
    return os.path.isfile(path_to_name)

def is_matching_file_in(path, name, glob):
    """Whether that name is a file in path and matches that glob"""
    return is_file_in(path, name) and fnmatch.fnmatch(name, glob)

def path_to_module(path, name):
    """Whether the name matches a python source or compiled file in that path

    If source and compiled files are found, give the source
    """
    glob = '%s.py*' % name
    python_files = [f
                    for f in directory_list(path)
                    if is_matching_file_in(path, f, glob)]
    if not python_files:
        return None
    source_files = [f for f in python_files if os.path.splitext(f)[-1] == '.py']
    python_file = source_files and source_files[0] or python_files[0]
    return os.path.join(path, python_file)

def path_to_sub_directory(path, name):
    """If name is a real sub-directory of path, return that"""
    result = os.path.join(path, name)
    if os.path.isdir(result):
        return result

def path_to_python(path, name):
    """Path to either a module or sub-dir of that path, with that name"""
    result = path_to_sub_directory(path, name)
    if result:
        return result
    return path_to_module(path, name)

def built_in(name):
    """Whether the name is that of one of python's builtin modules"""
    try:
        #  Not all builtin modules are initially imported, so bring it in first
        __import__(name)
    except ImportError:
        return False
    return '(built-in)' in str(sys.modules[name])

def path_to_import(string):
    module = importlib.import_module(string)
    if module:
        path_to_file = module.__file__
        if '.egg/' in path_to_file:
            return path_to_file.split('.egg/')[0] + '.egg'
        return path_to_file
    return None

def main(strings):
    """Run the program"""
    for string in strings:
        if built_in(string):
            print 'builtin', string
        path_to_imported_module = path_to_import(string)
        if path_to_imported_module:
            os.system('ls -ld %s' % path_to_imported_module)

if __name__ == '__main__':
    args = sys.argv[1:]
    if not args:
        from what import test
        sys.exit(test())
    else:
        sys.exit(main(args))
<|endoftext|>


<commit_before>#!/bin/env python3
# -*- coding: utf-8 -*-
#
# This file is part of the VecNet Zika modeling interface.
# For copyright and licensing information about this package, see the
# NOTICE.txt and LICENSE.txt files in its top-level directory; they are
# available at https://github.com/vecnet/zika
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License (MPL), version 2.0.  If a copy of the MPL was not distributed
# with this file, You can obtain one at http://mozilla.org/MPL/2.0/.

import csv
import logging

from django.core.exceptions import ObjectDoesNotExist
from django.db import connection
from django.utils.timezone import now

from website.apps.home.models import Location, Data, Simulation, SimulationModel, Totals, UploadJob

logger = logging.getLogger(__name__)

def load_simulation_file(upload_job):
    """
    :param upload_job:
    :type upload_job: website.apps.home.models.UploadJob
    :return:
    """
    # csvfile = upload_job.data_file.file
    # dialect = csv.Sniffer().sniff(csvfile.read(1024))
    # csvfile.seek(0)
    # Reopen in binary mode
    # upload_job.data_file.close()
    upload_job.status = UploadJob.IN_PROGRESS
    upload_job.upload_start_timestamp = now()
    upload_job.save(update_fields=['status', 'upload_start_timestamp'])
    try:
        csvfile = upload_job.data_file
        # Workaround for Bug #13809
        # https://code.djangoproject.com/ticket/13809
        # It is fixed in Django 1.11, so it will be possible to use upload_job.data_file.open("r")
        # But we are on Django 1.10 yet
        csvfile = upload_job.data_file.storage.open(upload_job.data_file.name, "r")
        in_memory_csv = []
        for row in csvfile:
            in_memory_csv.append(row)

        dictreader = csv.DictReader(in_memory_csv)

        line = None
        simulation_set = set()
        lines_read = 0

        for line in dictreader:
            lines_read += 1
            if lines_read % 100 == 0:
                upload_job.progress = int(100 * float(lines_read) / len(in_memory_csv))
                upload_job.save(update_fields=['progress'])
            if line['output_generate_date'] is None:
                # Assuming this is an empty line
                continue
            # Assuming date is in "YYYY-MM-DD HH:MM:SS" format
            # However, YYYY-MM-DD works, too
            date_output_generated = line['output_generate_date'].split(" ")[0]
            # Get or Create the SimulationModel object
            sim_model = SimulationModel.objects.get_or_create(model_name=line['model_name'].strip())[0]
            sim = Simulation.objects.filter(date_output_generated=date_output_generated, sim_model=sim_model).first()
            if sim and sim.is_uploaded:
                msg = 'Simulation %s has been uploaded already. Can\'t update.' % sim.id
                logger.error(msg)
                upload_job.status = UploadJob.FAILED
                upload_job.last_error_message = msg
                upload_job.save(update_fields=['status', 'last_error_message'])
                return False, msg
            if not sim:
                sim = Simulation(
                    name='%s %s' % (upload_job.name, str(date_output_generated)),
                    sim_model=sim_model,
                    date_output_generated=date_output_generated,
                    historical=upload_job.historical,
                    is_uploaded=False,
                    created_by=upload_job.created_by,
                )
                sim.save()

            location = Location.objects.filter(
                department_code=line['department_code'].strip(),
                municipality_code=line['municipality_code'].strip(),
            ).first()

            if not location:
                location = Location.objects.create(
                    department=line['department'].strip(),
                    department_code=line['department_code'].strip(),
                    municipality=line['municipality'].strip(),
                    municipality_code=line['municipality_code'].strip(),
                )

            if location.department != line['department']:
                print("WARNING: department name mismatch (%s, %s)" % (location.department, line['department']))

            if location.municipality != line['municipality']:
                print("WARNING: municipality name mismatch (%s, %s)" % (location.municipality, line['municipality']))
            simulation_set.add(sim)

            Data.objects.create(
                location=location,
                date=line['date'].split(" ")[0],  # Assuming date is in "YYYY-MM-DD HH:MM:SS" format
                simulation=sim,
                value_low=line['value_low'].strip(),
                value_mid=line['value_mid'].strip(),
                value_high=line['value_high'].strip(),
            )
            if connection.queries:
                print('Data: %s' % connection.queries[-1]['time'])
            try:
                total_obj = Totals.objects.get(data_date=line['date'].split(" ")[0], simulation=sim.id,
                                               date_output_generated=sim.date_output_generated)

                total_obj.total_low += float(line['value_low'])
                total_obj.total_mid += float(line['value_mid'])
                total_obj.total_high += float(line['value_high'])
                total_obj.save()
            except ObjectDoesNotExist:
                total = Totals.objects.create(
                    data_date=line['date'].split(" ")[0],
                    simulation=sim,
                    total_low=line['value_low'],
                    total_mid=line['value_mid'],
                    total_high=line['value_high'],
                    date_output_generated=sim.date_output_generated
                )
            if connection.queries:
                print('Totals: %s' % connection.queries[-1]['time'])
        for sim in simulation_set:
            # for data in Data.objects.filter(simulation=sim).distinct('date'):
            #
            sim.is_uploaded = True
            sim.save()
    except Exception as e:
        logger.exception('Exception: %s' % e)
        msg = '%s: %s' % ((type(e).__name__), str(e))
        upload_job.status = UploadJob.FAILED
        upload_job.last_error_message = msg
        upload_job.save(update_fields=['status', 'last_error_message'])
        return False, msg

    upload_job.status = UploadJob.COMPLETED
    upload_job.progress = 100
    upload_job.save(update_fields=['status', 'progress'])
    print('Upload complete')
    return True, ""
<commit_msg>More profiling code<commit_after>#!/bin/env python3
# -*- coding: utf-8 -*-
#
# This file is part of the VecNet Zika modeling interface.
# For copyright and licensing information about this package, see the
# NOTICE.txt and LICENSE.txt files in its top-level directory; they are
# available at https://github.com/vecnet/zika
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License (MPL), version 2.0.  If a copy of the MPL was not distributed
# with this file, You can obtain one at http://mozilla.org/MPL/2.0/.

import csv
import logging

from django.core.exceptions import ObjectDoesNotExist
from django.db import connection
from django.utils.timezone import now

from website.apps.home.models import Location, Data, Simulation, SimulationModel, Totals, UploadJob

logger = logging.getLogger(__name__)

def load_simulation_file(upload_job):
    """
    :param upload_job:
    :type upload_job: website.apps.home.models.UploadJob
    :return:
    """
    # csvfile = upload_job.data_file.file
    # dialect = csv.Sniffer().sniff(csvfile.read(1024))
    # csvfile.seek(0)
    # Reopen in binary mode
    # upload_job.data_file.close()
    upload_job.status = UploadJob.IN_PROGRESS
    upload_job.upload_start_timestamp = now()
    upload_job.save(update_fields=['status', 'upload_start_timestamp'])
    try:
        csvfile = upload_job.data_file
        # Workaround for Bug #13809
        # https://code.djangoproject.com/ticket/13809
        # It is fixed in Django 1.11, so it will be possible to use upload_job.data_file.open("r")
        # But we are on Django 1.10 yet
        csvfile = upload_job.data_file.storage.open(upload_job.data_file.name, "r")
        in_memory_csv = []
        for row in csvfile:
            in_memory_csv.append(row)

        dictreader = csv.DictReader(in_memory_csv)

        line = None
        simulation_set = set()
        lines_read = 0

        for line in dictreader:
            lines_read += 1
            if lines_read % 100 == 0:
                upload_job.progress = int(100 * float(lines_read) / len(in_memory_csv))
                upload_job.save(update_fields=['progress'])
            if line['output_generate_date'] is None:
                # Assuming this is an empty line
                continue
            # Assuming date is in "YYYY-MM-DD HH:MM:SS" format
            # However, YYYY-MM-DD works, too
            date_output_generated = line['output_generate_date'].split(" ")[0]
            # Get or Create the SimulationModel object
            sim_model = SimulationModel.objects.get_or_create(model_name=line['model_name'].strip())[0]
            sim = Simulation.objects.filter(date_output_generated=date_output_generated, sim_model=sim_model).first()
            if sim and sim.is_uploaded:
                msg = 'Simulation %s has been uploaded already. Can\'t update.' % sim.id
                logger.error(msg)
                upload_job.status = UploadJob.FAILED
                upload_job.last_error_message = msg
                upload_job.save(update_fields=['status', 'last_error_message'])
                return False, msg
            if not sim:
                sim = Simulation(
                    name='%s %s' % (upload_job.name, str(date_output_generated)),
                    sim_model=sim_model,
                    date_output_generated=date_output_generated,
                    historical=upload_job.historical,
                    is_uploaded=False,
                    created_by=upload_job.created_by,
                )
                sim.save()

            location = Location.objects.filter(
                department_code=line['department_code'].strip(),
                municipality_code=line['municipality_code'].strip(),
            ).first()
            if connection.queries:
                print('Location SELECT: %s' % connection.queries[-1]['time'])

            if not location:
                location = Location.objects.create(
                    department=line['department'].strip(),
                    department_code=line['department_code'].strip(),
                    municipality=line['municipality'].strip(),
                    municipality_code=line['municipality_code'].strip(),
                )

            if location.department != line['department']:
                print("WARNING: department name mismatch (%s, %s)" % (location.department, line['department']))

            if location.municipality != line['municipality']:
                print("WARNING: municipality name mismatch (%s, %s)" % (location.municipality, line['municipality']))
            simulation_set.add(sim)

            Data.objects.create(
                location=location,
                date=line['date'].split(" ")[0],  # Assuming date is in "YYYY-MM-DD HH:MM:SS" format
                simulation=sim,
                value_low=line['value_low'].strip(),
                value_mid=line['value_mid'].strip(),
                value_high=line['value_high'].strip(),
            )
            if connection.queries:
                print('Data: %s' % connection.queries[-1]['time'])
            try:
                total_obj = Totals.objects.get(data_date=line['date'].split(" ")[0], simulation=sim.id,
                                               date_output_generated=sim.date_output_generated)

                if connection.queries:
                    print('Totals SELECT: %s' % connection.queries[-1]['time'])
                total_obj.total_low += float(line['value_low'])
                total_obj.total_mid += float(line['value_mid'])
                total_obj.total_high += float(line['value_high'])
                total_obj.save()
            except ObjectDoesNotExist:
                total = Totals.objects.create(
                    data_date=line['date'].split(" ")[0],
                    simulation=sim,
                    total_low=line['value_low'],
                    total_mid=line['value_mid'],
                    total_high=line['value_high'],
                    date_output_generated=sim.date_output_generated
                )
            if connection.queries:
                print('Totals: %s' % connection.queries[-1]['time'])
        for sim in simulation_set:
            # for data in Data.objects.filter(simulation=sim).distinct('date'):
            #
            sim.is_uploaded = True
            sim.save()
    except Exception as e:
        logger.exception('Exception: %s' % e)
        msg = '%s: %s' % ((type(e).__name__), str(e))
        upload_job.status = UploadJob.FAILED
        upload_job.last_error_message = msg
        upload_job.save(update_fields=['status', 'last_error_message'])
        return False, msg

    upload_job.status = UploadJob.COMPLETED
    upload_job.progress = 100
    upload_job.save(update_fields=['status', 'progress'])
    print('Upload complete')
    return True, ""
<|endoftext|>


<commit_before>#!/usr/bin/env python
# vim: ai ts=4 sts=4 et sw=4

import os

DEBUG = True
TEMPLATE_DEBUG = DEBUG

ADMINS = (
    # ('Your Name', 'your_email@domain.com'),
)

MANAGERS = ADMINS

# Local time zone for this installation. Choices can be found here:
# http://en.wikipedia.org/wiki/List_of_tz_zones_by_name
# although not all choices may be available on all operating systems.
# If running in a Windows environment this must be set to the same as your
# system time zone.
TIME_ZONE = 'Africa/Lagos'

# Language code for this installation. All choices can be found here:
# http://www.i18nguy.com/unicode/language-identifiers.html
LANGUAGE_CODE = 'en-us'

SITE_ID = 1

# If you set this to False, Django will make some optimizations so as not
# to load the internationalization machinery.
USE_I18N = True

# Absolute path to the directory that holds media.
# Example: "/home/media/media.lawrence.com/"
MEDIA_ROOT = ''

# URL that handles the media served from MEDIA_ROOT. Make sure to use a
# trailing slash if there is a path component (optional in other cases).
# Examples: "http://media.lawrence.com", "http://example.com/media/"
MEDIA_URL = ''

# URL prefix for admin media -- CSS, JavaScript and images. Make sure to use a
# trailing slash.
# Examples: "http://foo.com/media/", "/media/".
ADMIN_MEDIA_PREFIX = '/media/'

# Make this unique, and don't share it with anybody.
SECRET_KEY = '2rgmwtyq$thj49+-6u7x9t39r7jflu&1ljj3x2c0n0fl$)04_0'

# List of callables that know how to import templates from various sources.
TEMPLATE_LOADERS = (
    'django.template.loaders.filesystem.load_template_source',
    'django.template.loaders.app_directories.load_template_source',
#     'django.template.loaders.eggs.load_template_source',
)

MIDDLEWARE_CLASSES = (
    'django.middleware.common.CommonMiddleware',
    'django.contrib.sessions.middleware.SessionMiddleware',
    'django.contrib.auth.middleware.AuthenticationMiddleware',
)

ROOT_URLCONF = "rapidsms.webui.urls"

TEMPLATE_CONTEXT_PROCESSORS = [
    "django.core.context_processors.auth",
    "django.core.context_processors.debug",
    "django.core.context_processors.i18n",
    "django.core.context_processors.media",
    "django.core.context_processors.request"
]

TEMPLATE_DIRS = [
    # Put strings here, like "/home/html/django_templates" or "C:/www/django/templates".
    # Always use forward slashes, even on Windows.
    # Don't forget to use absolute paths, not relative paths.
]

# ====================
# LOAD RAPIDSMS CONFIG
# ====================

# this module will usually be called up via the django
# reloader, which is (in turn) called up by rapidsms's
# server.py, which adds RAPIDSMS_INI to the environment.
# just in case, though, check that it's defined rather
# than repeating the filename selection logic here
if not "RAPIDSMS_INI" in os.environ:
    raise(
        EnvironmentError,
        "The RAPIDSMS_INI environment variable is not "  +\
        "defined. Without it, settings.py doesn't know " +\
        "which ini file to load settings from")

# load the rapidsms configuration
from rapidsms import Config
RAPIDSMS_CONF = Config(os.environ["RAPIDSMS_INI"])

# since iterating and reading the config of apps is
# common, build a handy dict of apps and their configs
RAPIDSMS_APPS = dict([
    (app["type"], app)
    for app in RAPIDSMS_CONF["rapidsms"]["apps"]])

# ==========================
# LOAD OTHER DJANGO SETTINGS
# ==========================

# load the database settings first, since those
# keys don't correspond exactly to their eventual
# name in this module (they're missing the prefix
# in rapidsms.ini); inject them into this module
if "database" in RAPIDSMS_CONF:
    for key, val in RAPIDSMS_CONF["database"].items():
        vars()["DATABASE_%s" % key.upper()] = val

else:
    # database settings are missing, so
    # blow up. TODO: is there a way to
    # run django without a database?
    raise(
        "Your RapidSMS configuration does not contain " +\
        "a [database] section, which is required " +\
        "for the Django WebUI to function.")

# if there is a "django" section, inject
# the items as CONSTANTS in this module
if "django" in RAPIDSMS_CONF:
    for key, val in RAPIDSMS_CONF["django"].items():
        vars()[key.upper()] = val

# ====================
# INJECT RAPIDSMS APPS
# ====================

INSTALLED_APPS = [
    'django.contrib.auth',
    'django.contrib.contenttypes',
    'django.contrib.sessions',
    'django.contrib.sites',
    'django.contrib.admin',
    'django.contrib.admindocs'
] + [app["module"] for app in RAPIDSMS_APPS.values()]
<commit_msg>use the system time zone unless overridden in rapidsms.ini<commit_after>#!/usr/bin/env python
# vim: ai ts=4 sts=4 et sw=4

import os, time

DEBUG = True
TEMPLATE_DEBUG = DEBUG

ADMINS = ()
MANAGERS = ADMINS

# default to the system's timezone settings. this can still be
# overridden in rapidsms.ini [django], by providing one of:
# http://en.wikipedia.org/wiki/List_of_tz_zones_by_name
TIME_ZONE = time.tzname[0]

# Language code for this installation. All choices can be found here:
# http://www.i18nguy.com/unicode/language-identifiers.html
LANGUAGE_CODE = 'en-us'

SITE_ID = 1

# If you set this to False, Django will make some optimizations so as not
# to load the internationalization machinery.
USE_I18N = True

# Absolute path to the directory that holds media.
# Example: "/home/media/media.lawrence.com/"
MEDIA_ROOT = ''

# URL that handles the media served from MEDIA_ROOT. Make sure to use a
# trailing slash if there is a path component (optional in other cases).
# Examples: "http://media.lawrence.com", "http://example.com/media/"
MEDIA_URL = ''

# URL prefix for admin media -- CSS, JavaScript and images. Make sure to use a
# trailing slash.
# Examples: "http://foo.com/media/", "/media/".
ADMIN_MEDIA_PREFIX = '/media/'

# Make this unique, and don't share it with anybody.
SECRET_KEY = '2rgmwtyq$thj49+-6u7x9t39r7jflu&1ljj3x2c0n0fl$)04_0'

# List of callables that know how to import templates from various sources.
TEMPLATE_LOADERS = (
    'django.template.loaders.filesystem.load_template_source',
    'django.template.loaders.app_directories.load_template_source',
#     'django.template.loaders.eggs.load_template_source',
)

MIDDLEWARE_CLASSES = (
    'django.middleware.common.CommonMiddleware',
    'django.contrib.sessions.middleware.SessionMiddleware',
    'django.contrib.auth.middleware.AuthenticationMiddleware',
)

ROOT_URLCONF = "rapidsms.webui.urls"

TEMPLATE_CONTEXT_PROCESSORS = [
    "django.core.context_processors.auth",
    "django.core.context_processors.debug",
    "django.core.context_processors.i18n",
    "django.core.context_processors.media",
    "django.core.context_processors.request"
]

TEMPLATE_DIRS = [
    # Put strings here, like "/home/html/django_templates" or "C:/www/django/templates".
    # Always use forward slashes, even on Windows.
    # Don't forget to use absolute paths, not relative paths.
]

# ====================
# LOAD RAPIDSMS CONFIG
# ====================

# this module will usually be called up via the django
# reloader, which is (in turn) called up by rapidsms's
# server.py, which adds RAPIDSMS_INI to the environment.
# just in case, though, check that it's defined rather
# than repeating the filename selection logic here
if not "RAPIDSMS_INI" in os.environ:
    raise(
        EnvironmentError,
        "The RAPIDSMS_INI environment variable is not "  +\
        "defined. Without it, settings.py doesn't know " +\
        "which ini file to load settings from")

# load the rapidsms configuration
from rapidsms import Config
RAPIDSMS_CONF = Config(os.environ["RAPIDSMS_INI"])

# since iterating and reading the config of apps is
# common, build a handy dict of apps and their configs
RAPIDSMS_APPS = dict([
    (app["type"], app)
    for app in RAPIDSMS_CONF["rapidsms"]["apps"]])

# ==========================
# LOAD OTHER DJANGO SETTINGS
# ==========================

# load the database settings first, since those
# keys don't correspond exactly to their eventual
# name in this module (they're missing the prefix
# in rapidsms.ini); inject them into this module
if "database" in RAPIDSMS_CONF:
    for key, val in RAPIDSMS_CONF["database"].items():
        vars()["DATABASE_%s" % key.upper()] = val

else:
    # database settings are missing, so
    # blow up. TODO: is there a way to
    # run django without a database?
    raise(
        "Your RapidSMS configuration does not contain " +\
        "a [database] section, which is required " +\
        "for the Django WebUI to function.")

# if there is a "django" section, inject
# the items as CONSTANTS in this module
if "django" in RAPIDSMS_CONF:
    for key, val in RAPIDSMS_CONF["django"].items():
        vars()[key.upper()] = val

# ====================
# INJECT RAPIDSMS APPS
# ====================

INSTALLED_APPS = [
    'django.contrib.auth',
    'django.contrib.contenttypes',
    'django.contrib.sessions',
    'django.contrib.sites',
    'django.contrib.admin',
    'django.contrib.admindocs'
] + [app["module"] for app in RAPIDSMS_APPS.values()]
<|endoftext|>


<commit_before>import sys
import os
from numpy import *
from scipy.linalg import *
from scipy.sparse import *
from scipy.io import *
from pyspark import SparkContext
import logging

if len(sys.argv) < 6:
  print >> sys.stderr, \
  "(sta) usage: delay <master> <inputFile_A> <inputFile_y> <outputFile> <mxLag>"
  exit(-1)

def parseVector(line):
	vec = [float(x) for x in line.split(' ')]
	ts = array(vec[3:])
	#ts = (ts - mean(ts))/std(ts)
	med = median(ts)
	ts = (ts - med) / (med)
	ts = (ts - mean(ts)) / std(ts)
	return ts

def getSta(x,y,lags):
	for i in arange(length(lags)):
		w[i] = mean(x * roll(y,int(lag[i])))
	return [dot(x,w)/sum(w),max(w)]

# parse inputs
sc = SparkContext(sys.argv[1], "delay")
inputFile_A = str(sys.argv[2])
inputFile_y = str(sys.argv[3])
mxLag = int(sys.argv[5])
outputFile = str(sys.argv[4]) + "-delay-mxLag-" + str(mxLag)
lags = arange(2*mxLag) - floor(2*mxLag/2)
if not os.path.exists(outputFile):
    os.makedirs(outputFile)
logging.basicConfig(filename=outputFile+'/'+'stdout.log',level=logging.INFO,format='%(asctime)s %(message)s',datefmt='%m/%d/%Y %I:%M:%S %p')

# parse data
logging.info("(delay) loading data")
lines_A = sc.textFile(inputFile_A)
lines_y = sc.textFile(inputFile_y)
y = array([float(x) for x in lines_y.collect()[0].split(' ')])
y = (y - mean(y))/std(y)
A = lines_A.map(parseVector).cache()

# compute sta
logging.info('(delay) computing delay'
sta = A.map(lambda x : getSta(x,y,lags))
logging.info('(delay) saving results...')
savemat(outputFile+"/"+"ph.mat",mdict={'ph':sta.map(lambda x : x[0]).collect()},oned_as='column',do_compression='true')
savemat(outputFile+"/"+"co.mat",mdict={'co':sta.map(lambda x : x[1]).collect()},oned_as='column',do_compression='true')
<commit_msg>Bug fixes<commit_after>import sys
import os
from numpy import *
from scipy.linalg import *
from scipy.sparse import *
from scipy.io import *
from pyspark import SparkContext
import logging

if len(sys.argv) < 6:
  print >> sys.stderr, \
  "(sta) usage: delay <master> <inputFile_A> <inputFile_y> <outputFile> <mxLag>"
  exit(-1)

def parseVector(line):
	vec = [float(x) for x in line.split(' ')]
	ts = array(vec[3:])
	#ts = (ts - mean(ts))/std(ts)
	med = median(ts)
	ts = (ts - med) / (med)
	ts = (ts - mean(ts)) / std(ts)
	return ts

def getSta(x,y,lags):
	for i in arange(length(lags)):
		w[i] = mean(x * roll(y,int(lag[i])))
	return [dot(x,w)/sum(w),max(w)]

# parse inputs
sc = SparkContext(sys.argv[1], "delay")
inputFile_A = str(sys.argv[2])
inputFile_y = str(sys.argv[3])
mxLag = int(sys.argv[5])
outputFile = str(sys.argv[4]) + "-delay-mxLag-" + str(mxLag)
lags = arange(2*mxLag) - floor(2*mxLag/2)
if not os.path.exists(outputFile):
    os.makedirs(outputFile)
logging.basicConfig(filename=outputFile+'/'+'stdout.log',level=logging.INFO,format='%(asctime)s %(message)s',datefmt='%m/%d/%Y %I:%M:%S %p')

# parse data
logging.info("(delay) loading data")
lines_A = sc.textFile(inputFile_A)
lines_y = sc.textFile(inputFile_y)
y = array([float(x) for x in lines_y.collect()[0].split(' ')])
y = (y - mean(y))/std(y)
A = lines_A.map(parseVector).cache()

# compute sta
logging.info('(delay) computing delay')
sta = A.map(lambda x : getSta(x,y,lags))
logging.info('(delay) saving results...')
savemat(outputFile+"/"+"ph.mat",mdict={'ph':sta.map(lambda x : x[0]).collect()},oned_as='column',do_compression='true')
savemat(outputFile+"/"+"co.mat",mdict={'co':sta.map(lambda x : x[1]).collect()},oned_as='column',do_compression='true')
<|endoftext|>


<commit_before><commit_msg>only call setraw if the screen is talking to a tty<commit_after><|endoftext|>


<commit_before># Copyright (c) 2012 Web Notes Technologies Pvt Ltd.
# License: GNU General Public License (v3). For more information see license.txt

from __future__ import unicode_literals

import webnotes
from webnotes.utils import cstr
from website.utils import build_html, url_for_website, delete_page_cache

@webnotes.whitelist(allow_guest=True)
def get_product_info(item_code):
	"""get product price / stock info"""
	price_list = webnotes.conn.get_value("Item", item_code, "website_price_list")
	warehouse = webnotes.conn.get_value("Item", item_code, "website_warehouse")
	if warehouse:
		in_stock = webnotes.conn.sql("""select actual_qty from tabBin where
			item_code=%s and warehouse=%s""", (item_code, warehouse))
		if in_stock:
			in_stock = in_stock[0][0] > 0 and 1 or 0
	else:
		in_stock = -1
	return {
		"price": price_list and webnotes.conn.sql("""select ref_rate, ref_currency from
			`tabItem Price` where parent=%s and price_list_name=%s""",
			(item_code, price_list), as_dict=1) or [],
		"stock": in_stock
	}

@webnotes.whitelist(allow_guest=True)
def get_product_list(search=None, product_group=None, start=0, limit=10):
	# DOUBT: why is product_group param passed?

	# base query
	query = """select name, item_name, page_name, website_image, item_group,
			web_long_description as website_description
		from `tabItem` where docstatus = 0 and show_in_website = 1 """

	# search term condition
	if search:
		query += """and (web_long_description like %(search)s or
				item_name like %(search)s or name like %(search)s)"""
		search = "%" + cstr(search) + "%"

	# order by
	query += """order by weightage desc, modified desc limit %s, %s""" % (start, limit)

	data = webnotes.conn.sql(query, {
		"search": search,
		"product_group": product_group
	}, as_dict=1)

	return [get_item_for_list_in_html(r) for r in data]

def get_product_list_for_group(product_group=None, start=0, limit=10):
	child_groups = ", ".join(['"' + i[0] + '"' for i in get_child_groups(product_group)])

	# base query
	query = """select name, item_name, page_name, website_image, item_group,
			web_long_description as website_description
		from `tabItem` where docstatus = 0 and show_in_website = 1
		and (item_group in (%s)
			or name in (select parent from `tabWebsite Item Group` where item_group in (%s))) """ % (child_groups, child_groups)

	query += """order by weightage desc, modified desc limit %s, %s""" % (start, limit)

	data = webnotes.conn.sql(query, {"product_group": product_group}, as_dict=1)

	return [get_item_for_list_in_html(r) for r in data]

def get_child_groups(item_group_name):
	item_group = webnotes.doc("Item Group", item_group_name)
	return webnotes.conn.sql("""select name
		from `tabItem Group` where lft>=%(lft)s and rgt<=%(rgt)s
			and show_in_website = 1""", item_group.fields)

def get_group_item_count(item_group):
	child_groups = ", ".join(['"' + i[0] + '"' for i in get_child_groups(item_group)])
	return webnotes.conn.sql("""select count(*) from `tabItem`
		where docstatus = 0 and show_in_website = 1
		and (item_group in (%s)
			or name in (select parent from `tabWebsite Item Group`
				where item_group in (%s))) """ % (child_groups, child_groups))[0][0]

def get_item_for_list_in_html(r):
	scrub_item_for_list(r)
	r.template = "html/product_in_list.html"
	return build_html(r)

def scrub_item_for_list(r):
	if not r.website_description:
		r.website_description = "No description given"
	r.website_image = url_for_website(r.website_image)

def get_parent_item_groups(item_group_name):
	item_group = webnotes.doc("Item Group", item_group_name)
	return webnotes.conn.sql("""select name, page_name from `tabItem Group`
		where lft <= %s and rgt >= %s
		and ifnull(show_in_website,0)=1
		order by lft asc""", (item_group.lft, item_group.rgt), as_dict=True)

def invalidate_cache_for(item_group):
	for i in get_parent_item_groups(item_group):
		if i.page_name:
			delete_page_cache(i.page_name)<commit_msg>removed product param in get_product_list<commit_after># Copyright (c) 2012 Web Notes Technologies Pvt Ltd.
# License: GNU General Public License (v3). For more information see license.txt

from __future__ import unicode_literals

import webnotes
from webnotes.utils import cstr
from website.utils import build_html, url_for_website, delete_page_cache

@webnotes.whitelist(allow_guest=True)
def get_product_info(item_code):
	"""get product price / stock info"""
	price_list = webnotes.conn.get_value("Item", item_code, "website_price_list")
	warehouse = webnotes.conn.get_value("Item", item_code, "website_warehouse")
	if warehouse:
		in_stock = webnotes.conn.sql("""select actual_qty from tabBin where
			item_code=%s and warehouse=%s""", (item_code, warehouse))
		if in_stock:
			in_stock = in_stock[0][0] > 0 and 1 or 0
	else:
		in_stock = -1
	return {
		"price": price_list and webnotes.conn.sql("""select ref_rate, ref_currency from
			`tabItem Price` where parent=%s and price_list_name=%s""",
			(item_code, price_list), as_dict=1) or [],
		"stock": in_stock
	}

@webnotes.whitelist(allow_guest=True)
def get_product_list(search=None, start=0, limit=10):
	# base query
	query = """select name, item_name, page_name, website_image, item_group,
			web_long_description as website_description
		from `tabItem` where docstatus = 0 and show_in_website = 1 """

	# search term condition
	if search:
		query += """and (web_long_description like %(search)s or
				item_name like %(search)s or name like %(search)s)"""
		search = "%" + cstr(search) + "%"

	# order by
	query += """order by weightage desc, modified desc limit %s, %s""" % (start, limit)

	data = webnotes.conn.sql(query, {
		"search": search,
	}, as_dict=1)

	return [get_item_for_list_in_html(r) for r in data]

def get_product_list_for_group(product_group=None, start=0, limit=10):
	child_groups = ", ".join(['"' + i[0] + '"' for i in get_child_groups(product_group)])

	# base query
	query = """select name, item_name, page_name, website_image, item_group,
			web_long_description as website_description
		from `tabItem` where docstatus = 0 and show_in_website = 1
		and (item_group in (%s)
			or name in (select parent from `tabWebsite Item Group` where item_group in (%s))) """ % (child_groups, child_groups)

	query += """order by weightage desc, modified desc limit %s, %s""" % (start, limit)

	data = webnotes.conn.sql(query, {"product_group": product_group}, as_dict=1)

	return [get_item_for_list_in_html(r) for r in data]

def get_child_groups(item_group_name):
	item_group = webnotes.doc("Item Group", item_group_name)
	return webnotes.conn.sql("""select name
		from `tabItem Group` where lft>=%(lft)s and rgt<=%(rgt)s
			and show_in_website = 1""", item_group.fields)

def get_group_item_count(item_group):
	child_groups = ", ".join(['"' + i[0] + '"' for i in get_child_groups(item_group)])
	return webnotes.conn.sql("""select count(*) from `tabItem`
		where docstatus = 0 and show_in_website = 1
		and (item_group in (%s)
			or name in (select parent from `tabWebsite Item Group`
				where item_group in (%s))) """ % (child_groups, child_groups))[0][0]

def get_item_for_list_in_html(r):
	scrub_item_for_list(r)
	r.template = "html/product_in_list.html"
	return build_html(r)

def scrub_item_for_list(r):
	if not r.website_description:
		r.website_description = "No description given"
	r.website_image = url_for_website(r.website_image)

def get_parent_item_groups(item_group_name):
	item_group = webnotes.doc("Item Group", item_group_name)
	return webnotes.conn.sql("""select name, page_name from `tabItem Group`
		where lft <= %s and rgt >= %s
		and ifnull(show_in_website,0)=1
		order by lft asc""", (item_group.lft, item_group.rgt), as_dict=True)

def invalidate_cache_for(item_group):
	for i in get_parent_item_groups(item_group):
		if i.page_name:
			delete_page_cache(i.page_name)<|endoftext|>


<commit_before># Copyright 2014 Knowledge Economy Developments Ltd
#
# Henry Gomersall
# heng@kedevelopments.co.uk
#
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
# * Redistributions of source code must retain the above copyright notice, this
# list of conditions and the following disclaimer.
#
# * Redistributions in binary form must reproduce the above copyright notice,
# this list of conditions and the following disclaimer in the documentation
# and/or other materials provided with the distribution.
#
# * Neither the name of the copyright holder nor the names of its contributors
# may be used to endorse or promote products derived from this software without
# specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.
#

from pyfftw.interfaces import scipy_fftpack
import pyfftw
import numpy
import scipy
import scipy.fftpack
import scipy.signal

import unittest
from .test_pyfftw_base import run_test_suites
from . import test_pyfftw_numpy_interface

'''pyfftw.interfaces.scipy_fftpack just wraps pyfftw.interfaces.numpy_fft.

All the tests here just check that the call is made correctly.
'''

funcs = ('fft','ifft', 'fft2', 'ifft2', 'fftn', 'ifftn',
           'rfft', 'irfft')

acquired_names = ('dct', 'idct', 'diff', 'tilbert', 'itilbert', 'hilbert',
        'ihilbert', 'cs_diff', 'sc_diff', 'ss_diff', 'cc_diff', 'shift',
        'fftshift', 'ifftshift', 'fftfreq', 'rfftfreq', 'convolve',
        '_fftpack')

def make_complex_data(shape, dtype):
    ar, ai = dtype(numpy.random.randn(2, *shape))
    return ar + 1j*ai

def make_r2c_real_data(shape, dtype):
    return dtype(numpy.random.randn(*shape))

def make_c2r_real_data(shape, dtype):
    return dtype(numpy.random.randn(*shape))

make_complex_data = test_pyfftw_numpy_interface.make_complex_data

complex_dtypes = test_pyfftw_numpy_interface.complex_dtypes
real_dtypes = test_pyfftw_numpy_interface.real_dtypes

def numpy_fft_replacement(a, s, axes, overwrite_input, planner_effort,
        threads, auto_align_input, auto_contiguous):

    return (a, s, axes, overwrite_input, planner_effort,
        threads, auto_align_input, auto_contiguous)

io_dtypes = {
        'complex': (complex_dtypes, make_complex_data),
        'r2c': (real_dtypes, make_r2c_real_data),
        'c2r': (real_dtypes, make_c2r_real_data)}

class InterfacesScipyFFTPackTestSimple(unittest.TestCase):
    ''' A really simple test suite to check simple implementation.
    '''

    def test_scipy_overwrite(self):
        scipy_fftn = scipy.signal.signaltools.fftn
        scipy_ifftn = scipy.signal.signaltools.ifftn

        a = pyfftw.n_byte_align_empty((128, 64), 16, dtype='complex128')
        b = pyfftw.n_byte_align_empty((128, 64), 16, dtype='complex128')

        a[:] = (numpy.random.randn(*a.shape) +
                1j*numpy.random.randn(*a.shape))
        b[:] = (numpy.random.randn(*b.shape) +
                1j*numpy.random.randn(*b.shape))

        scipy_c = scipy.signal.fftconvolve(a, b)

        scipy.signal.signaltools.fftn = scipy_fftpack.fftn
        scipy.signal.signaltools.ifftn = scipy_fftpack.ifftn

        scipy_replaced_c = scipy.signal.fftconvolve(a, b)

        self.assertTrue(numpy.allclose(scipy_c, scipy_replaced_c))

        scipy.signal.signaltools.fftn = scipy_fftn
        scipy.signal.signaltools.ifftn = scipy_ifftn

    def test_funcs(self):

        for each_func in funcs:
            func_being_replaced = getattr(scipy_fftpack, each_func)

            #create args (8 of them)
            args = []
            for n in range(8):
                args.append(object())

            args = tuple(args)

            try:
                setattr(scipy_fftpack, each_func,
                        numpy_fft_replacement)

                return_args = getattr(scipy_fftpack, each_func)(*args)
                for n, each_arg in enumerate(args):
                    # Check that what comes back is what is sent
                    # (which it should be)
                    self.assertIs(each_arg, return_args[n])
            except:
                raise

            finally:
                setattr(scipy_fftpack, each_func,
                        func_being_replaced)

    def test_acquired_names(self):
        for each_name in acquired_names:

            fftpack_attr = getattr(scipy.fftpack, each_name)
            acquired_attr = getattr(scipy_fftpack, each_name)

            self.assertIs(fftpack_attr, acquired_attr)

# Construct all the test classes automatically.
built_classes = []
for each_func in funcs:

    class_name = 'InterfacesScipyFFTPackTest' + each_func.upper()

    parent_class_name = 'InterfacesNumpyFFTTest' + each_func.upper()
    parent_class = getattr(test_pyfftw_numpy_interface, parent_class_name)

    class_dict = {'validator_module': scipy.fftpack,
                'test_interface': scipy_fftpack,
                'io_dtypes': io_dtypes,
                'overwrite_input_flag': 'overwrite_x',
                'default_s_from_shape_slicer': slice(None)}

    globals()[class_name] = type(class_name,
            (parent_class,), class_dict)

    built_classes.append(globals()[class_name])

built_classes = tuple(built_classes)

test_cases = (
        InterfacesScipyFFTPackTestSimple,) + built_classes

test_set = None
#test_set = {'InterfacesScipyFFTPackTestIFFTN': ['test_auto_align_input']}

if __name__ == '__main__':

    run_test_suites(test_cases, test_set)
<commit_msg>Added a skip for the unittests when scipy is missing.<commit_after># Copyright 2014 Knowledge Economy Developments Ltd
#
# Henry Gomersall
# heng@kedevelopments.co.uk
#
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
# * Redistributions of source code must retain the above copyright notice, this
# list of conditions and the following disclaimer.
#
# * Redistributions in binary form must reproduce the above copyright notice,
# this list of conditions and the following disclaimer in the documentation
# and/or other materials provided with the distribution.
#
# * Neither the name of the copyright holder nor the names of its contributors
# may be used to endorse or promote products derived from this software without
# specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.
#

from pyfftw.interfaces import scipy_fftpack
import pyfftw
import numpy

try:
    import scipy
    import scipy.fftpack
    import scipy.signal

except ImportError:
    scipy_missing = True

else:
    scipy_missing = False

import unittest
from .test_pyfftw_base import run_test_suites
from . import test_pyfftw_numpy_interface

'''pyfftw.interfaces.scipy_fftpack just wraps pyfftw.interfaces.numpy_fft.

All the tests here just check that the call is made correctly.
'''

funcs = ('fft','ifft', 'fft2', 'ifft2', 'fftn', 'ifftn',
           'rfft', 'irfft')

acquired_names = ('dct', 'idct', 'diff', 'tilbert', 'itilbert', 'hilbert',
        'ihilbert', 'cs_diff', 'sc_diff', 'ss_diff', 'cc_diff', 'shift',
        'fftshift', 'ifftshift', 'fftfreq', 'rfftfreq', 'convolve',
        '_fftpack')

def make_complex_data(shape, dtype):
    ar, ai = dtype(numpy.random.randn(2, *shape))
    return ar + 1j*ai

def make_r2c_real_data(shape, dtype):
    return dtype(numpy.random.randn(*shape))

def make_c2r_real_data(shape, dtype):
    return dtype(numpy.random.randn(*shape))

make_complex_data = test_pyfftw_numpy_interface.make_complex_data

complex_dtypes = test_pyfftw_numpy_interface.complex_dtypes
real_dtypes = test_pyfftw_numpy_interface.real_dtypes

def numpy_fft_replacement(a, s, axes, overwrite_input, planner_effort,
        threads, auto_align_input, auto_contiguous):

    return (a, s, axes, overwrite_input, planner_effort,
        threads, auto_align_input, auto_contiguous)

io_dtypes = {
        'complex': (complex_dtypes, make_complex_data),
        'r2c': (real_dtypes, make_r2c_real_data),
        'c2r': (real_dtypes, make_c2r_real_data)}

@unittest.skipIf(scipy_missing, 'scipy is not installed, so this feature is'
                 'unavailable')
class InterfacesScipyFFTPackTestSimple(unittest.TestCase):
    ''' A really simple test suite to check simple implementation.
    '''

    def test_scipy_overwrite(self):
        scipy_fftn = scipy.signal.signaltools.fftn
        scipy_ifftn = scipy.signal.signaltools.ifftn

        a = pyfftw.n_byte_align_empty((128, 64), 16, dtype='complex128')
        b = pyfftw.n_byte_align_empty((128, 64), 16, dtype='complex128')

        a[:] = (numpy.random.randn(*a.shape) +
                1j*numpy.random.randn(*a.shape))
        b[:] = (numpy.random.randn(*b.shape) +
                1j*numpy.random.randn(*b.shape))

        scipy_c = scipy.signal.fftconvolve(a, b)

        scipy.signal.signaltools.fftn = scipy_fftpack.fftn
        scipy.signal.signaltools.ifftn = scipy_fftpack.ifftn

        scipy_replaced_c = scipy.signal.fftconvolve(a, b)

        self.assertTrue(numpy.allclose(scipy_c, scipy_replaced_c))

        scipy.signal.signaltools.fftn = scipy_fftn
        scipy.signal.signaltools.ifftn = scipy_ifftn

    def test_funcs(self):

        for each_func in funcs:
            func_being_replaced = getattr(scipy_fftpack, each_func)

            #create args (8 of them)
            args = []
            for n in range(8):
                args.append(object())

            args = tuple(args)

            try:
                setattr(scipy_fftpack, each_func,
                        numpy_fft_replacement)

                return_args = getattr(scipy_fftpack, each_func)(*args)
                for n, each_arg in enumerate(args):
                    # Check that what comes back is what is sent
                    # (which it should be)
                    self.assertIs(each_arg, return_args[n])
            except:
                raise

            finally:
                setattr(scipy_fftpack, each_func,
                        func_being_replaced)

    def test_acquired_names(self):
        for each_name in acquired_names:

            fftpack_attr = getattr(scipy.fftpack, each_name)
            acquired_attr = getattr(scipy_fftpack, each_name)

            self.assertIs(fftpack_attr, acquired_attr)

# Construct all the test classes automatically.
built_classes = []
for each_func in funcs:

    class_name = 'InterfacesScipyFFTPackTest' + each_func.upper()

    parent_class_name = 'InterfacesNumpyFFTTest' + each_func.upper()
    parent_class = getattr(test_pyfftw_numpy_interface, parent_class_name)

    class_dict = {'validator_module': scipy.fftpack,
                'test_interface': scipy_fftpack,
                'io_dtypes': io_dtypes,
                'overwrite_input_flag': 'overwrite_x',
                'default_s_from_shape_slicer': slice(None)}

    globals()[class_name] = type(class_name,
            (parent_class,), class_dict)

    built_classes.append(globals()[class_name])

built_classes = tuple(built_classes)

test_cases = (
        InterfacesScipyFFTPackTestSimple,) + built_classes

test_set = None
#test_set = {'InterfacesScipyFFTPackTestIFFTN': ['test_auto_align_input']}

if __name__ == '__main__':

    run_test_suites(test_cases, test_set)
<|endoftext|>