<commit_before>from __future__ import print_function
import numpy as np
from scipy.sparse.linalg import eigs

def rv_pca(data, n_datasets):
    """
    Get weights for tables by calculating their similarity.

    :return:
    """

    print("Rv-PCA")
    C = np.zeros([n_datasets, n_datasets])

    print("Rv-PCA: Hilbert-Schmidt inner products... ", end='')
    for i in range(n_datasets):
        for j in range(n_datasets):
            C[i, j] = np.sum(data[i].affinity_ * data[j].affinity_)

    print('Done!')

    print("Rv-PCA: Decomposing the inner product matrix... ", end ='')
    e, u = eigs(C, k=8)
    print("Done!")
    weights = (np.real(u[:,0]) / np.sum(np.real(u[:,0]))).flatten()

    print("Rv-PCA: Done!")

    return weights, np.real(e), np.real(u)

def aniso_c1(X, M):
    """
    Calculate weights using ANISOSTATIS criterion 1
    :param X:
    :param M:
    :return: New weights
    """
    print("Computing ANISOSTATIS Criterion 1 weights...", end='')

    C = np.power(X.T.dot(M.dot(X)), 2)
    ev, Mn = eigs(C, k=1)
    Mn = np.real(Mn)
    Mn = np.concatenate(Mn / np.sum(Mn))

    print('Done!')
    return Mn, ev

def get_A_STATIS(data, table_weights, n_datasets):
    """
    Dataset masses.

    """

    print("Dataset/variable masses... ", end='')
    a = np.concatenate([np.repeat(table_weights[i], data[i].n_var) for i in range(n_datasets)])

    return np.diag(a)

def get_A_ANISOSTATIS(table_weights):

    return np.diag(table_weights)

def get_M(n_obs):
    """
    Masses for observations. These are assumed to be equal.
    """

    print("Observation masses: Done!")

    return np.eye(n_obs) / n_obs

def stack_tables(data, n_datasets):
    """
    Stacks preprocessed tables horizontally
    """

    print("Stack datasets for GSVD...", end='')

    X = np.concatenate([data[i].data_std_ for i in range(n_datasets)], axis=1)
    X_scaled = np.concatenate([data[i].data_scaled_ for i in range(n_datasets)], axis=1)

    print("Done!")
    return X, X_scaled

def get_col_indices(data, ids, groups, ugroups):

    """
    Returns dict(s) that maps IDs to columns

    :return:
    """

    print('Getting indices... ', end = '')

    col_indices = []
    c = 0
    for i, u in enumerate(ids):
        col_indices.append(np.arange(c, c + data[i].n_var))
        c += data[i].n_var

    grp_indices = []
    for i, ug in enumerate(ugroups):
        ginds = []
        for g in ug:
            ginds.append(np.concatenate(list(map(col_indices.__getitem__, np.where(groups[:, i] == g)[0]))))
        grp_indices.append(ginds)

    print('Done!')

    return col_indices, grp_indices

def gsvd(X, M, A, n_comps = 10):
    """
    Generalized SVD

    :param X:
    :param M:
    :param A:
    :return:
    """

    print("GSVD")
    print("GSVD: Weights... ", end='')
    Xw = np.dot(np.sqrt(M), np.dot(X, np.sqrt(A)))
    print("Done!")

    print("GSVD: SVD... ", end='')
    [P_, D, Q_] = np.linalg.svd(Xw, full_matrices=False)

    P_ = P_[:,0:n_comps]
    D = D[0:n_comps]
    Q_ = Q_[0:n_comps,:]
    print('Done!')

    print("GSVD: Factor scores and eigenvalues... ", end='')
    Mp = np.power(np.diag(M), -0.5)
    Ap = np.power(np.diag(A), -0.5)

    P = np.dot(np.diag(Mp), P_)
    Q = np.dot(np.diag(Ap), Q_.T)
    ev = np.power(D, 2)

    print('Done!')

    return P, D, Q, ev<commit_msg>Calculate only a few components instead of the whole space<commit_after>from __future__ import print_function
import numpy as np
from scipy.sparse.linalg import eigs
from sklearn.utils.extmath import randomized_svd

def rv_pca(data, n_datasets):
    """
    Get weights for tables by calculating their similarity.

    :return:
    """

    print("Rv-PCA")
    C = np.zeros([n_datasets, n_datasets])

    print("Rv-PCA: Hilbert-Schmidt inner products... ", end='')
    for i in range(n_datasets):
        for j in range(n_datasets):
            C[i, j] = np.sum(data[i].affinity_ * data[j].affinity_)

    print('Done!')

    print("Rv-PCA: Decomposing the inner product matrix... ", end ='')
    e, u = eigs(C, k=8)
    print("Done!")
    weights = (np.real(u[:,0]) / np.sum(np.real(u[:,0]))).flatten()

    print("Rv-PCA: Done!")

    return weights, np.real(e), np.real(u)

def aniso_c1(X, M):
    """
    Calculate weights using ANISOSTATIS criterion 1
    :param X:
    :param M:
    :return: New weights
    """
    print("Computing ANISOSTATIS Criterion 1 weights...", end='')

    C = np.power(X.T.dot(M.dot(X)), 2)
    ev, Mn = eigs(C, k=1)
    Mn = np.real(Mn)
    Mn = np.concatenate(Mn / np.sum(Mn))

    print('Done!')
    return Mn, ev

def get_A_STATIS(data, table_weights, n_datasets):
    """
    Dataset masses.

    """

    print("Dataset/variable masses... ", end='')
    a = np.concatenate([np.repeat(table_weights[i], data[i].n_var) for i in range(n_datasets)])

    return np.diag(a)

def get_A_ANISOSTATIS(table_weights):

    return np.diag(table_weights)

def get_M(n_obs):
    """
    Masses for observations. These are assumed to be equal.
    """

    print("Observation masses: Done!")

    return np.eye(n_obs) / n_obs

def stack_tables(data, n_datasets):
    """
    Stacks preprocessed tables horizontally
    """

    print("Stack datasets for GSVD...", end='')

    X = np.concatenate([data[i].data_std_ for i in range(n_datasets)], axis=1)
    X_scaled = np.concatenate([data[i].data_scaled_ for i in range(n_datasets)], axis=1)

    print("Done!")
    return X, X_scaled

def get_col_indices(data, ids, groups, ugroups):

    """
    Returns dict(s) that maps IDs to columns

    :return:
    """

    print('Getting indices... ', end = '')

    col_indices = []
    c = 0
    for i, u in enumerate(ids):
        col_indices.append(np.arange(c, c + data[i].n_var))
        c += data[i].n_var

    grp_indices = []
    for i, ug in enumerate(ugroups):
        ginds = []
        for g in ug:
            ginds.append(np.concatenate(list(map(col_indices.__getitem__, np.where(groups[:, i] == g)[0]))))
        grp_indices.append(ginds)

    print('Done!')

    return col_indices, grp_indices

def gsvd(X, M, A, n_comps = 10):
    """
    Generalized SVD

    :param X:
    :param M:
    :param A:
    :return:
    """

    print("GSVD")
    print("GSVD: Weights... ", end='')
    Xw = np.dot(np.sqrt(M), np.dot(X, np.sqrt(A)))
    print("Done!")

    print("GSVD: SVD... ", end='')
    [P_, D, Q_] = randomized_svd(Xw, n_comps)

    #P_ = P_[:,0:n_comps]
    #D = D[0:n_comps]
    #Q_ = Q_[0:n_comps,:]
    print('Done!')

    print("GSVD: Factor scores and eigenvalues... ", end='')
    Mp = np.power(np.diag(M), -0.5)
    Ap = np.power(np.diag(A), -0.5)

    P = np.dot(np.diag(Mp), P_)
    Q = np.dot(np.diag(Ap), Q_.T)
    ev = np.power(D, 2)

    print('Done!')

    return P, D, Q, ev<|endoftext|>


<commit_before>#
# Copyright (c) 2004 Specifix, Inc.
# All rights reserved
#
import imp, sys
import os
import util
import build
import package
import shutil
import types
import inspect
import lookaside
import rpmhelper
import gzip

baseMacros = (
    # Note that these macros cannot be represented as a dictionary,
    # because the items need to be added in order so that they will
    # be properly interpolated.
    #
    # paths
    ('prefix'		, '/usr'),
    ('sysconfdir'	, '/etc'),
    ('lib'              , 'lib'),  # may be overridden with 'lib64'
    ('exec_prefix'	, '%(prefix)s'),
    ('bindir'		, '%(exec_prefix)s/bin'),
    ('sbindir'		, '%(exec_prefix)s/sbin'),
    ('libdir'		, '%(exec_prefix)s/%(lib)s'),
    ('libexecdir'	, '%(exec_prefix)s/libexec'),
    ('localstatedir'	, '%(prefix)s/var'),
    ('sharedstatedir'	, '%(prefix)s/com'),
    ('includedir'	, '%(prefix)s/include'),
    ('datadir'		, '/usr/share'),
    ('mandir'		, '%(datadir)s/man'),
    ('infodir'		, '%(datadir)s/info'),
    ('docdir'		, '%(datadir)s/doc'),
    ('develdocdir'	, '%(datadir)s/develdoc'),
    # arguments/flags (empty ones are for documentation; non-existant = empty)
    ('cflags'           , '-O2'),
    ('mflags'		, ''),
    ('parallelmflags'   , ''),
    ('sysroot'		, ''),
)

crossMacros = (
    # set crossdir from cook, directly or indirectly, before adding the rest
    #('crossdir'	, 'cross-target'),
    ('prefix'		, '/opt/%(crossdir)s'),
    ('sysroot'		, '%(prefix)s/sys-root'),
    ('headerpath'	, '%(sysroot)s/usr/include')
)

baseAutoSpec = (
    # automatic subpackage names and sets of regexps that define them
    # cannot be a dictionary because it is ordered; first match wins
    ('devel',
	('\.a',
	 '\.so',
	 '.*/include/.*\.h',
	 '/usr/include/',
	 '/usr/share/man/man(2|3)/',
	 '/usr/share/develdoc/')),
    ('lib', ('.*/lib/.*\.so\..*')),
    ('doc', ('/usr/share/(doc|man|info)/')),
    ('locale', ('/usr/share/locale/',)),
    ('runtime', ('.*',)),
)

class Macros(dict):
    def __setitem__(self, name, value):
	dict.__setitem__(self, name, value % self)

    # we want keys that don't exist to default to empty strings
    def __getitem__(self, name):
	if self.has_key(name):
	    return dict.__getitem__(self, name)
	return ''

    def addMacros(self, *macroSet):
	# must be in order; later macros in the set can depend on
	# earlier ones
	# for ease of use, we allow passing in a tuple of tuples, or
	# a simple set of tuples
	if len(macroSet) == 1 and type(macroSet[0]) is tuple:
	    # we were passed a tuple of tuples (like baseMacros)
	    macroSet = macroSet[0]
        if len(macroSet) > 0 and type(macroSet[0]) is not tuple:
            # we were passed something like ('foo', 'bar')
            macroSet = (macroSet,)
	for key, value in macroSet:
	    self[key] = value

    def copy(self):
	new = Macros()
	new.update(self)
	return new

def flatten(list):
    if type(list) != types.ListType: return [list]
    if list == []: return list
    return flatten(list[0]) + flatten(list[1:])

def extractSourceFromRPM(rpm, targetfile):
    filename = os.path.basename(targetfile)
    directory = os.path.dirname(targetfile)
    r = file(rpm, 'r')
    rpmhelper.seekToData(r)
    gz = gzip.GzipFile(fileobj=r)
    (rpipe, wpipe) = os.pipe()
    pid = os.fork()
    if not pid:
	os.dup2(rpipe, 0)
	os.chdir(directory)
	os.execl('/bin/cpio', 'cpio', '-ium', filename)
	sys.exit(1)
    while 1:
	buf = gz.read(4096)
	if not buf:
	    break
	os.write(wpipe, buf)
    os.close(wpipe)
    (pid, status) = os.waitpid(pid, 0)
    if not os.WIFEXITED(status):
	raise IOError, 'cpio died extracting %s from RPM %s' \
	               %(filename, os.path.basename(rpm))
    if os.WEXITSTATUS(status):
	raise IOError, 'cpio returned failure %d extracting %s from RPM %s' \
	               %(os.WEXITSTATUS(status), filename, os.path.basename(rpm))
    if not os.path.exists(targetfile):
	raise IOError, 'failed to extract source %s from RPM %s' \
	               %(filename, os.path.basename(rpm))

class RecipeLoader(types.DictionaryType):
    def __init__(self, file):
        if file[0] != "/":
            raise IOError, "recipe file names must be absolute paths"

        self.file = os.path.basename(file).replace('.', '-')
        self.module = imp.new_module(self.file)
        sys.modules[self.file] = self.module
        f = open(file)

        exec 'from recipe import Recipe' in self.module.__dict__
        exec 'from recipe import loadRecipe' in self.module.__dict__
        exec 'import build, os, package, sys, util' in self.module.__dict__
        if sys.excepthook == util.excepthook:
            exec 'sys.excepthook = util.excepthook' in self.module.__dict__
        exec 'filename = "%s"' %(file) in self.module.__dict__
        code = compile(f.read(), file, 'exec')
        exec code in self.module.__dict__
        for (name, obj) in  self.module.__dict__.items():
            if type(obj) == types.ClassType:
                # make sure the class is derived from Recipe
                # and has a name
                if obj.__dict__.has_key('ignore'):
                    continue
                if issubclass(obj, Recipe) and obj.__dict__.has_key('name'):
                    obj.__dict__['filename'] = file
                    self[name] = obj

    def __del__(self):
        try:
            del sys.modules[self.file]
        except:
            pass

# XXX this should be extended to load a recipe from srs
def loadRecipe(file):
    callerGlobals = inspect.stack()[1][0].f_globals
    if file[0] != '/':
        recipepath = os.path.dirname(callerGlobals['filename'])
        file = recipepath + '/' + file
    recipes = RecipeLoader(file)
    for name, recipe in recipes.items():
        # XXX hack to hide parent recipes
        recipe.ignore = 1
        callerGlobals[name] = recipe
        # stash a reference to the module in the namespace
        # of the recipe that loaded it, or else it will be destroyed
        callerGlobals[os.path.basename(file).replace('.', '-')] = recipes

#def bootstrapRecipe(file, class, buildRequires):
#    loadRecipe(file) # XXX not necessary if we put boostraps in main files
#    exec """class Bootstrap%s(%s):
#	buildRequires = %s
#	name = "bootstrap-%s"
#	def setup(self):
#	    FIXMEcrossmacros(self.recipeCfg)
#	    FIXMEcrossenv
#	    FIXMEself.mainDir(class, self.version)
#	    %s.setup(self)
#    """ %(class, class, buildRequires.repr(), class, class)

class Recipe:
    buildRequires = []
    runRequires = []

    def addSignature(self, file, keyid):
	# do not search unless a gpg keyid is specified
	if not keyid:
	    return
	gpg = '%s.sig' %(file)
	c = lookaside.searchAll(self.cfg, self.laReposCache, gpg,
				self.name, self.srcdirs)
	if not c:
	    gpg = '%s.sign' %(file)
	    c = lookaside.searchAll(self.cfg, self.laReposCache,
				    gpg, self.name, self.srcdirs)
	if c:
	    if not self.signatures.has_key(file):
		self.signatures[file] = []
	    self.signatures[file].append((gpg, c, keyid))

    def addTarball(self, file, extractDir='', keyid=None):
	self.tarballs.append((file, extractDir))
	self.addSignature(file, keyid)

    def addTarballFromRPM(self, rpm, file, extractDir='', keyid=None):
	f = lookaside.searchAll(self.cfg, self.laReposCache,
			     os.path.basename(file), self.name, self.srcdirs)
	if not f:
	    r = lookaside.findAll(self.cfg, self.laReposCache, rpm,
				  self.name, self.srcdirs)
	    c = lookaside.createCacheName(self.cfg, file, self.name)
	    extractSourceFromRPM(r, c)
	    f = lookaside.findAll(self.cfg, self.laReposCache, file,
				  self.name, self.srcdirs)
	self.tarballs.append((file, extractDir))
	self.addSignature(f, keyid)

    def addPatch(self, file, level='1', backup='', keyid=None):
	self.patches.append((file, level, backup))
	self.addSignature(file, keyid)

    def addSource(self, file, keyid=None):
	self.sources.append(file)
	self.addSignature(file, keyid)

    def allSources(self):
        sources = []
        for (tarball, extractdir) in self.tarballs:
            sources.append(tarball)
        for (patch, level, backup) in self.patches:
            sources.append(patch)
	for signaturelist in self.signatures.values():
            for (gpg, cached, keyid) in signaturelist:
                sources.append(gpg)
	return sources + self.sources

    def mainDir(self, new = None):
	if new:
	    self.theMainDir = new

	return self.theMainDir

    def nameVer(self):
	return self.name + "-" + self.version

    def cleanup(self, builddir, destdir):
	shutil.rmtree(builddir)
	shutil.rmtree(destdir)

    def checkSignatures(self, filepath, file):
        if not self.signatures.has_key(file):
            return
	for (gpg, signature, keyid) in self.signatures[file]:
	    # FIXME: our own keyring
	    if os.system("gpg --no-secmem-warning --verify %s %s"
			  %(signature, filepath)):
		# FIXME: only do this if key missing, this is cheap for now
		os.system("gpg --keyserver pgp.mit.edu --recv-keys 0x %s"
		          %(keyid))
		if os.system("gpg --no-secmem-warning --verify %s %s"
			      %(signature, filepath)):
		    raise RuntimeError, "GPG signature %s failed" %(signature)

    def unpackSources(self, builddir):
	if os.path.exists(builddir):
	    shutil.rmtree(builddir)
	util.mkdirChain(builddir)
	for (file, extractdir) in self.tarballs:
            f = lookaside.findAll(self.cfg, self.laReposCache, file,
				  self.name, self.srcdirs)
	    self.checkSignatures(f, file)
            if f.endswith(".bz2"):
                tarflags = "-jxf"
            elif f.endswith(".gz") or f.endswith(".tgz"):
                tarflags = "-zxf"
            else:
                raise RuntimeError, "unknown archive compression"
            if extractdir:
                destdir = '%s/%s' % (builddir, extractdir)
                util.execute("mkdir -p %s" % destdir)
            else:
                destdir = builddir
            util.execute("tar -C %s %s %s" % (destdir, tarflags, f))

	for file in self.sources:
            f = lookaside.findAll(self.cfg, self.laReposCache, file,
				  self.name, self.srcdirs)
	    destDir = builddir + "/" + self.theMainDir
	    util.mkdirChain(destDir)
	    shutil.copyfile(f, destDir + "/" + file)

	for (file, level, backup) in self.patches:
            # XXX handle .gz/.bz2 patch files
            f = util.findFile(file, self.srcdirs)
	    destDir = builddir + "/" + self.theMainDir
            if backup:
                backup = '-b -z %s' % backup
            util.execute('patch -d %s -p%s %s < %s' %(destDir, level, backup, f))

    def doBuild(self, buildpath):
        builddir = buildpath + "/" + self.mainDir()
	self.macros['builddir'] = builddir
        if self.build is None:
            pass
        elif type(self.build) is str:
            util.execute(self.build %self.macros)
        elif type(self.build) is tuple:
	    for bld in self.build:
                if type(bld) is str:
                    util.execute(bld %self.macros)
                else:
                    bld.doBuild(self.macros)
	else:
	    self.build.doBuild(self.macros)

    def doInstall(self, buildpath, root):
        builddir = buildpath + "/" + self.mainDir()
	self.addMacros(('builddir', builddir),
	               ('destdir', root))
        if self.install is None:
            pass
        elif type(self.install) is str:
            util.execute(self.install %self.macros)
	elif type(self.install) is tuple:
	    for inst in self.install:
                if type(inst) is str:
                    util.execute(inst %self.macros)
                else:
                    inst.doInstall(self.macros)
	else:
	    self.install.doInstall(self.macros)

    def packages(self, root):
	self.autoSpecList = []
	for spec in baseAutoSpec:
	    self.autoSpecList.append(package.PackageSpec(spec[0], spec[1]))
	# "None" will be replaced by explicit subpackage list
	self.packageSpecSet = package.PackageSpecSet(self.autoSpecList, None)
        self.packageSet = package.Auto(self.name, root, self.packageSpecSet)

    def getPackageSet(self):
        return self.packageSet

    def __init__(self, cfg, laReposCache, srcdirs, extraMacros=()):
	self.tarballs = []
	self.patches = []
	self.sources = []
	self.signatures = {}
        self.cfg = cfg
	self.laReposCache = laReposCache
	self.srcdirs = srcdirs
	self.theMainDir = self.name + "-" + self.version
	self.build = build.Make()
        self.install = build.MakeInstall()
	self.macros = Macros()
	self.addMacros = self.macros.addMacros
	self.addMacros(baseMacros)
	self.macros['name'] = self.name
	self.macros['version'] = self.version
	if extraMacros:
	    self.addMacros(extraMacros)
<commit_msg>datadir should follow prefix, how did I get this wrong?<commit_after>#
# Copyright (c) 2004 Specifix, Inc.
# All rights reserved
#
import imp, sys
import os
import util
import build
import package
import shutil
import types
import inspect
import lookaside
import rpmhelper
import gzip

baseMacros = (
    # Note that these macros cannot be represented as a dictionary,
    # because the items need to be added in order so that they will
    # be properly interpolated.
    #
    # paths
    ('prefix'		, '/usr'),
    ('sysconfdir'	, '/etc'),
    ('lib'              , 'lib'),  # may be overridden with 'lib64'
    ('exec_prefix'	, '%(prefix)s'),
    ('bindir'		, '%(exec_prefix)s/bin'),
    ('sbindir'		, '%(exec_prefix)s/sbin'),
    ('libdir'		, '%(exec_prefix)s/%(lib)s'),
    ('libexecdir'	, '%(exec_prefix)s/libexec'),
    ('localstatedir'	, '%(prefix)s/var'),
    ('sharedstatedir'	, '%(prefix)s/com'),
    ('includedir'	, '%(prefix)s/include'),
    ('datadir'		, '%(prefix)s/share'),
    ('mandir'		, '%(datadir)s/man'),
    ('infodir'		, '%(datadir)s/info'),
    ('docdir'		, '%(datadir)s/doc'),
    ('develdocdir'	, '%(datadir)s/develdoc'),
    # arguments/flags (empty ones are for documentation; non-existant = empty)
    ('cflags'           , '-O2'),
    ('mflags'		, ''),
    ('parallelmflags'   , ''),
    ('sysroot'		, ''),
)

crossMacros = (
    # set crossdir from cook, directly or indirectly, before adding the rest
    #('crossdir'	, 'cross-target'),
    ('prefix'		, '/opt/%(crossdir)s'),
    ('sysroot'		, '%(prefix)s/sys-root'),
    ('headerpath'	, '%(sysroot)s/usr/include')
)

baseAutoSpec = (
    # automatic subpackage names and sets of regexps that define them
    # cannot be a dictionary because it is ordered; first match wins
    ('devel',
	('\.a',
	 '\.so',
	 '.*/include/.*\.h',
	 '/usr/include/',
	 '/usr/share/man/man(2|3)/',
	 '/usr/share/develdoc/')),
    ('lib', ('.*/lib/.*\.so\..*')),
    ('doc', ('/usr/share/(doc|man|info)/')),
    ('locale', ('/usr/share/locale/',)),
    ('runtime', ('.*',)),
)

class Macros(dict):
    def __setitem__(self, name, value):
	dict.__setitem__(self, name, value % self)

    # we want keys that don't exist to default to empty strings
    def __getitem__(self, name):
	if self.has_key(name):
	    return dict.__getitem__(self, name)
	return ''

    def addMacros(self, *macroSet):
	# must be in order; later macros in the set can depend on
	# earlier ones
	# for ease of use, we allow passing in a tuple of tuples, or
	# a simple set of tuples
	if len(macroSet) == 1 and type(macroSet[0]) is tuple:
	    # we were passed a tuple of tuples (like baseMacros)
	    macroSet = macroSet[0]
        if len(macroSet) > 0 and type(macroSet[0]) is not tuple:
            # we were passed something like ('foo', 'bar')
            macroSet = (macroSet,)
	for key, value in macroSet:
	    self[key] = value

    def copy(self):
	new = Macros()
	new.update(self)
	return new

def flatten(list):
    if type(list) != types.ListType: return [list]
    if list == []: return list
    return flatten(list[0]) + flatten(list[1:])

def extractSourceFromRPM(rpm, targetfile):
    filename = os.path.basename(targetfile)
    directory = os.path.dirname(targetfile)
    r = file(rpm, 'r')
    rpmhelper.seekToData(r)
    gz = gzip.GzipFile(fileobj=r)
    (rpipe, wpipe) = os.pipe()
    pid = os.fork()
    if not pid:
	os.dup2(rpipe, 0)
	os.chdir(directory)
	os.execl('/bin/cpio', 'cpio', '-ium', filename)
	sys.exit(1)
    while 1:
	buf = gz.read(4096)
	if not buf:
	    break
	os.write(wpipe, buf)
    os.close(wpipe)
    (pid, status) = os.waitpid(pid, 0)
    if not os.WIFEXITED(status):
	raise IOError, 'cpio died extracting %s from RPM %s' \
	               %(filename, os.path.basename(rpm))
    if os.WEXITSTATUS(status):
	raise IOError, 'cpio returned failure %d extracting %s from RPM %s' \
	               %(os.WEXITSTATUS(status), filename, os.path.basename(rpm))
    if not os.path.exists(targetfile):
	raise IOError, 'failed to extract source %s from RPM %s' \
	               %(filename, os.path.basename(rpm))

class RecipeLoader(types.DictionaryType):
    def __init__(self, file):
        if file[0] != "/":
            raise IOError, "recipe file names must be absolute paths"

        self.file = os.path.basename(file).replace('.', '-')
        self.module = imp.new_module(self.file)
        sys.modules[self.file] = self.module
        f = open(file)

        exec 'from recipe import Recipe' in self.module.__dict__
        exec 'from recipe import loadRecipe' in self.module.__dict__
        exec 'import build, os, package, sys, util' in self.module.__dict__
        if sys.excepthook == util.excepthook:
            exec 'sys.excepthook = util.excepthook' in self.module.__dict__
        exec 'filename = "%s"' %(file) in self.module.__dict__
        code = compile(f.read(), file, 'exec')
        exec code in self.module.__dict__
        for (name, obj) in  self.module.__dict__.items():
            if type(obj) == types.ClassType:
                # make sure the class is derived from Recipe
                # and has a name
                if obj.__dict__.has_key('ignore'):
                    continue
                if issubclass(obj, Recipe) and obj.__dict__.has_key('name'):
                    obj.__dict__['filename'] = file
                    self[name] = obj

    def __del__(self):
        try:
            del sys.modules[self.file]
        except:
            pass

# XXX this should be extended to load a recipe from srs
def loadRecipe(file):
    callerGlobals = inspect.stack()[1][0].f_globals
    if file[0] != '/':
        recipepath = os.path.dirname(callerGlobals['filename'])
        file = recipepath + '/' + file
    recipes = RecipeLoader(file)
    for name, recipe in recipes.items():
        # XXX hack to hide parent recipes
        recipe.ignore = 1
        callerGlobals[name] = recipe
        # stash a reference to the module in the namespace
        # of the recipe that loaded it, or else it will be destroyed
        callerGlobals[os.path.basename(file).replace('.', '-')] = recipes

#def bootstrapRecipe(file, class, buildRequires):
#    loadRecipe(file) # XXX not necessary if we put boostraps in main files
#    exec """class Bootstrap%s(%s):
#	buildRequires = %s
#	name = "bootstrap-%s"
#	def setup(self):
#	    FIXMEcrossmacros(self.recipeCfg)
#	    FIXMEcrossenv
#	    FIXMEself.mainDir(class, self.version)
#	    %s.setup(self)
#    """ %(class, class, buildRequires.repr(), class, class)

class Recipe:
    buildRequires = []
    runRequires = []

    def addSignature(self, file, keyid):
	# do not search unless a gpg keyid is specified
	if not keyid:
	    return
	gpg = '%s.sig' %(file)
	c = lookaside.searchAll(self.cfg, self.laReposCache, gpg,
				self.name, self.srcdirs)
	if not c:
	    gpg = '%s.sign' %(file)
	    c = lookaside.searchAll(self.cfg, self.laReposCache,
				    gpg, self.name, self.srcdirs)
	if c:
	    if not self.signatures.has_key(file):
		self.signatures[file] = []
	    self.signatures[file].append((gpg, c, keyid))

    def addTarball(self, file, extractDir='', keyid=None):
	self.tarballs.append((file, extractDir))
	self.addSignature(file, keyid)

    def addTarballFromRPM(self, rpm, file, extractDir='', keyid=None):
	f = lookaside.searchAll(self.cfg, self.laReposCache,
			     os.path.basename(file), self.name, self.srcdirs)
	if not f:
	    r = lookaside.findAll(self.cfg, self.laReposCache, rpm,
				  self.name, self.srcdirs)
	    c = lookaside.createCacheName(self.cfg, file, self.name)
	    extractSourceFromRPM(r, c)
	    f = lookaside.findAll(self.cfg, self.laReposCache, file,
				  self.name, self.srcdirs)
	self.tarballs.append((file, extractDir))
	self.addSignature(f, keyid)

    def addPatch(self, file, level='1', backup='', keyid=None):
	self.patches.append((file, level, backup))
	self.addSignature(file, keyid)

    def addSource(self, file, keyid=None):
	self.sources.append(file)
	self.addSignature(file, keyid)

    def allSources(self):
        sources = []
        for (tarball, extractdir) in self.tarballs:
            sources.append(tarball)
        for (patch, level, backup) in self.patches:
            sources.append(patch)
	for signaturelist in self.signatures.values():
            for (gpg, cached, keyid) in signaturelist:
                sources.append(gpg)
	return sources + self.sources

    def mainDir(self, new = None):
	if new:
	    self.theMainDir = new

	return self.theMainDir

    def nameVer(self):
	return self.name + "-" + self.version

    def cleanup(self, builddir, destdir):
	shutil.rmtree(builddir)
	shutil.rmtree(destdir)

    def checkSignatures(self, filepath, file):
        if not self.signatures.has_key(file):
            return
	for (gpg, signature, keyid) in self.signatures[file]:
	    # FIXME: our own keyring
	    if os.system("gpg --no-secmem-warning --verify %s %s"
			  %(signature, filepath)):
		# FIXME: only do this if key missing, this is cheap for now
		os.system("gpg --keyserver pgp.mit.edu --recv-keys 0x %s"
		          %(keyid))
		if os.system("gpg --no-secmem-warning --verify %s %s"
			      %(signature, filepath)):
		    raise RuntimeError, "GPG signature %s failed" %(signature)

    def unpackSources(self, builddir):
	if os.path.exists(builddir):
	    shutil.rmtree(builddir)
	util.mkdirChain(builddir)
	for (file, extractdir) in self.tarballs:
            f = lookaside.findAll(self.cfg, self.laReposCache, file,
				  self.name, self.srcdirs)
	    self.checkSignatures(f, file)
            if f.endswith(".bz2"):
                tarflags = "-jxf"
            elif f.endswith(".gz") or f.endswith(".tgz"):
                tarflags = "-zxf"
            else:
                raise RuntimeError, "unknown archive compression"
            if extractdir:
                destdir = '%s/%s' % (builddir, extractdir)
                util.execute("mkdir -p %s" % destdir)
            else:
                destdir = builddir
            util.execute("tar -C %s %s %s" % (destdir, tarflags, f))

	for file in self.sources:
            f = lookaside.findAll(self.cfg, self.laReposCache, file,
				  self.name, self.srcdirs)
	    destDir = builddir + "/" + self.theMainDir
	    util.mkdirChain(destDir)
	    shutil.copyfile(f, destDir + "/" + file)

	for (file, level, backup) in self.patches:
            # XXX handle .gz/.bz2 patch files
            f = util.findFile(file, self.srcdirs)
	    destDir = builddir + "/" + self.theMainDir
            if backup:
                backup = '-b -z %s' % backup
            util.execute('patch -d %s -p%s %s < %s' %(destDir, level, backup, f))

    def doBuild(self, buildpath):
        builddir = buildpath + "/" + self.mainDir()
	self.macros['builddir'] = builddir
        if self.build is None:
            pass
        elif type(self.build) is str:
            util.execute(self.build %self.macros)
        elif type(self.build) is tuple:
	    for bld in self.build:
                if type(bld) is str:
                    util.execute(bld %self.macros)
                else:
                    bld.doBuild(self.macros)
	else:
	    self.build.doBuild(self.macros)

    def doInstall(self, buildpath, root):
        builddir = buildpath + "/" + self.mainDir()
	self.addMacros(('builddir', builddir),
	               ('destdir', root))
        if self.install is None:
            pass
        elif type(self.install) is str:
            util.execute(self.install %self.macros)
	elif type(self.install) is tuple:
	    for inst in self.install:
                if type(inst) is str:
                    util.execute(inst %self.macros)
                else:
                    inst.doInstall(self.macros)
	else:
	    self.install.doInstall(self.macros)

    def packages(self, root):
	self.autoSpecList = []
	for spec in baseAutoSpec:
	    self.autoSpecList.append(package.PackageSpec(spec[0], spec[1]))
	# "None" will be replaced by explicit subpackage list
	self.packageSpecSet = package.PackageSpecSet(self.autoSpecList, None)
        self.packageSet = package.Auto(self.name, root, self.packageSpecSet)

    def getPackageSet(self):
        return self.packageSet

    def __init__(self, cfg, laReposCache, srcdirs, extraMacros=()):
	self.tarballs = []
	self.patches = []
	self.sources = []
	self.signatures = {}
        self.cfg = cfg
	self.laReposCache = laReposCache
	self.srcdirs = srcdirs
	self.theMainDir = self.name + "-" + self.version
	self.build = build.Make()
        self.install = build.MakeInstall()
	self.macros = Macros()
	self.addMacros = self.macros.addMacros
	self.addMacros(baseMacros)
	self.macros['name'] = self.name
	self.macros['version'] = self.version
	if extraMacros:
	    self.addMacros(extraMacros)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
#!/usr/bin/python
# Author: Niam Moltta
# UY - 2017
# MIT License
import math
import re
import numpy as np
import pandas as pd

print ' '
print ' '
print '                Welcome to RangeR.py'
print '                -- by Niam Moltta --'
print ' '
print ' '

fhand = raw_input('Enter file name: ')

if fhand == '':
    exit()

filecsv = str(fhand)

data = pd.read_csv(filecsv)

print ' '

frame = pd.DataFrame(data)

colist = frame.columns
columns = np.asarray(colist)

print columns
print ' '
hand = raw_input('Enter column header to apply ranges: ')
print ' '
puntajede = data[hand]

puntajes = np.asarray(puntajede)

scoresD = list()
scores = list()

total = 0 # control

range1 = 0
range2 = 0
range3 = 0
range4 = 0
range5 = 0
range55 = 0
range6 = 0
range7 = 0
range8 = 0
range9 = 0
range10 = 0

newScore = list() #ranged values

for line in puntajes:
    punts = float(line)
    scoresD.append(punts)

for value in scoresD :
    scores.append(float(value))

print ' '
print 'Values to assign:', len(scores)
print ' '

while True:

    for score in scores:
        value = float(score)
        if 14.041 < value <= 45.6542:
            val = 1
            newScore.append(val)
            range1 = range1 + 1
            total = total +1

        elif 45.6542 < value <= 49.935018:
            val = 2
            newScore.append(val)
            range2 = range2 + 1
            total = total +1

        elif 49.935018 < value <= 50.099664:
            val = 3
            newScore.append(val)
            range3 = range3 + 1
            total = total +1

        elif 50.099664 < value <= 50.264311: # negatives range reference
            val = 4
            newScore.append(val)
            range4 = range4 + 1
            total = total +1

        elif 50.264311 < value <= 66.729:
            val = 5
            newScore.append(val)
            range5 = range5 + 1
            total = total + 1

        else:
            print 'value could not be assigned:', value
            continue
    break

print ' '
print 'Ranges succesfully applied, total reduced to', total, 'values.'
print ' '
print 'Number of scores in range 1 =', range1
print 'Number of scores in range 2 =', range2
print 'Number of scores in range 3 =', range3
print 'Number of scores in range 4 =', range4
print 'Number of scores in range 5 =', range5
'''print 'Number of scores in range 5.5', range55
print 'Number of scores in range 6 =', range6
print 'Number of scores in range 7 =', range7
print 'Number of scores in range 8 =', range8
print 'Number of scores in range 9 =', range9
print 'Number of scores in range 10 =', range10'''

print ' '
print 'Number of values assigned to newScore list =', len(newScore)
print ' '

nfile = open('Ranges.txt', 'w')

for value in newScore:
    valor = str(value)
    nfile.write(valor)
    nfile.write('\n')

nfile.close()

print 'File created as "Ranges.txt"'
print ' '
<commit_msg>Delete RangeR.py<commit_after><|endoftext|>


<commit_before>import get_clean as gc
import get_code as gd
import test as test
import numpy as np
import pandas as pd

good=[]

total=0
can_try=0
test_try=0
diff_list=[]

def cal(code):
	try:
		a=gc.getData(code)
	except:
		print('{} is wrong'.format(code))
	else:
		print('{} is running'.format(code))
		if a is not None and len(a)>60:
			global total
			total=total+1
			a.sort_index(inplace=True)
			gc.ma(a,'close',[5,10,15,20,25])
			MA_column=a.columns[-5:]
			a['Diff']= 100 * ((a[MA_column].max(axis=1) - a[MA_column].min(axis=1))/a[MA_column].max(axis=1))
			b=a[-65:-5]['Diff'].mean()
			if b<2 and a[-65:-5]['Diff'].max()<10:
				good.append(code)
				global can_try
				can_try=can_try+1
				# print('{},{}'.format(a[-2:-1].index.values[0],a[-1:].index.values[0]))
				diff=test.test(code,a[-6:-5].index.values[0],a[-1:].index.values[0])
				diff_list.append(diff)
				if diff>0:
					global test_try
					test_try=test_try+1

def get_low(code):
	try:
		a=gc.getData(code)
	except:
		print('{} is wrong'.format(code))
	else:
		print('{} is running'.format(code))
		if a is not None:
			a.sort_index(inplace=True)
			gc.ma(a,'close',[15,30,60,90,120])
			MA_column=a.columns[-5:]
			a['Diff']=100 * ((a[MA_column].max(axis=1) - a[MA_column].min(axis=1))/a[MA_column].max(axis=1))
			a['Min']= a[MA_column].min(axis=1)
			b=a[-2:-1]
			if b['Min'].values[0] == b['MA_15'].values[0] and b['Diff'].values[0] > 40:
				good.append(code)

def clean_data(code):
	i=30
	j=0
	a=gc.cleanData(code)
	if a is not None:
		while i<len(a):
			if (a.iloc[i+7:i+8]['close'].get_values()-a.iloc[i:i+1]['close'].get_values())/a.iloc[i:i+1]['close'].get_values() >0.2:
				print('code : {} times : {}'.format(code,j))
				datas=a.iloc[i:i+1]
				datas['code']=code
				datas['target']=1
				datas.to_csv('data.csv',mode='a',header=False)
				i=i+8
				j=j+1
				good=datas.columns
			else:
				if np.random.random()>0.3:
					datas=a.iloc[i:i+1]
					datas['code']=code
					datas['target']=0
					print(datas.shape)
					print(datas.columns)
					datas.to_csv('data.csv',mode='a',header=False)
				i=i+1

good=[]

def kdj(data,date,m1,m2):
	data_use=data[['high','low','open','close']]
	data['lown'] = pd.rolling_min(data_use['low'], date)
	data.lown.fillna(value=pd.expanding_min(data_use['low']), inplace=True)
	data['highn'] = pd.rolling_max(data_use['high'], date)
	data.highn.fillna(value=pd.expanding_max(data_use['high']), inplace=True)
	data['rsv']=(data['close'] - data['lown']) / (data['highn'] - data['lown']) * 100
	data['kdj_k'] = pd.ewma(data['rsv'], m1)
	data['kdj_d'] = pd.ewma(data['kdj_k'], m2)
	data['kdj_j'] = 3 * data['kdj_k'] - 2 * data['kdj_d']

def kdj_k_min_10(code):
	a=gc.cleanData(code)
	if a is not None:
		if a.ix[-1:].kdj_k.get_values()<10:
			good.append(code)

def calculateLocation(now,max,min):
	mean=(max+min)/2
	return np.sign(now-mean)*(np.sqrt((now-mean)**2)/mean)+1

needCol=['MA_15','MA_30', 'MA_60', 'MA_90', 'MA_120']
def fiveLine(code):
	try:
		a=gc.getData(code)
	except:
		print('{} is wrong'.format(code))
	else:
		print('{} is running'.format(code))
		if a is not None:
			a.sort_index(inplace=True)
			gc.ma(a,'close',[15,30,60,90,120])
			return a

def calculateFive(a,needCol):
	std=a[needCol].std(axis=1).get_values()
	max=a[needCol].max(axis=1).get_values()
	min=a[needCol].min(axis=1).get_values()
	val_15=a.MA_15.get_values()
	val_now=a.close.get_values()
	loc_15=calculateLocation(a.MA_15,max,min).get_values()
	loc_now=calculateLocation(a.close,a.MA_15,a.MA_15).get_values()
	print('{} , {} , {} , {}'.format(val_15,val_now,loc_now,std))
	if std<0.3 and val_15<max:
		if loc_now>1.02 and val_now<max:
			return True

code=gd.get_all_code()

def useFive(code):
	a=fiveLine(code)
	global can_try
	global test_try
	if a is not None and len(a)>30:
		for i in range(30,a.shape[0]):
			if calculateFive(a.iloc[i:i+1],needCol):
				can_try+=1
				if a.iloc[i+5:i+6]['close'].get_values()>a.iloc[i:i+1]['close'].get_values():
					test_try+=1
					good.append((a.iloc[i+5:i+6]['close'].get_values()-a.iloc[i:i+1]['close'].get_values())/a.iloc[i:i+1]['close'].get_values())

# for j in code:
# 	kdj_k_min_10(j)

for j in code:
	useFive(j)
print(can_try)
print(test_try)

print(good)
print(np.mean(good))
# print('total:{},try:{},good:{}'.format(total,can_try,test_try))

# if test_try/can_try>0.5:
# 	x=np.array(diff_list)
# 	print('this strage max:{},min:{},mean:{},std:{}'.format(x.max(),x.min(),x.mean(),x.std()))

# print(good)
# 	get_low(code)

# print(good)

# good=['600234', '600959', '601016', '601021', '603019', '603021', '603028', '603029', '603085', '603117', '603131', '603169', '603223', '603315', '603318', '603338', '603528', '603588', '603598', '603601', '603663', '603678', '603686', '603703', '603779', '603818', '603869', '603959', '000897', '002143', '002195', '002240', '002354', '002520', '002577', '002602', '002681', '002739', '002743', '002746', '002747', '002751', '002752', '002771', '002773', '002777', '002780', '002785', '002796', '002798', '002799', '002800', '002801', '002803', '002805', '002806']

# j=0

# for i in good:
# 	diff=test.test(i,'2016-09-19','2016-09-20')
# 	if diff>0:
# 		j=j+1

# print('total:{},good:{}'.format(len(good),j))		<commit_msg>add get five<commit_after>import get_clean as gc
import get_code as gd
import test as test
import numpy as np
import pandas as pd

good=[]

total=0
can_try=0
test_try=0
diff_list=[]

def cal(code):
	try:
		a=gc.getData(code)
	except:
		print('{} is wrong'.format(code))
	else:
		print('{} is running'.format(code))
		if a is not None and len(a)>60:
			global total
			total=total+1
			a.sort_index(inplace=True)
			gc.ma(a,'close',[5,10,15,20,25])
			MA_column=a.columns[-5:]
			a['Diff']= 100 * ((a[MA_column].max(axis=1) - a[MA_column].min(axis=1))/a[MA_column].max(axis=1))
			b=a[-65:-5]['Diff'].mean()
			if b<2 and a[-65:-5]['Diff'].max()<10:
				good.append(code)
				global can_try
				can_try=can_try+1
				# print('{},{}'.format(a[-2:-1].index.values[0],a[-1:].index.values[0]))
				diff=test.test(code,a[-6:-5].index.values[0],a[-1:].index.values[0])
				diff_list.append(diff)
				if diff>0:
					global test_try
					test_try=test_try+1

def get_low(code):
	try:
		a=gc.getData(code)
	except:
		print('{} is wrong'.format(code))
	else:
		print('{} is running'.format(code))
		if a is not None:
			a.sort_index(inplace=True)
			gc.ma(a,'close',[15,30,60,90,120])
			MA_column=a.columns[-5:]
			a['Diff']=100 * ((a[MA_column].max(axis=1) - a[MA_column].min(axis=1))/a[MA_column].max(axis=1))
			a['Min']= a[MA_column].min(axis=1)
			b=a[-2:-1]
			if b['Min'].values[0] == b['MA_15'].values[0] and b['Diff'].values[0] > 40:
				good.append(code)

def clean_data(code):
	i=30
	j=0
	a=gc.cleanData(code)
	if a is not None:
		while i<len(a):
			if (a.iloc[i+7:i+8]['close'].get_values()-a.iloc[i:i+1]['close'].get_values())/a.iloc[i:i+1]['close'].get_values() >0.2:
				print('code : {} times : {}'.format(code,j))
				datas=a.iloc[i:i+1]
				datas['code']=code
				datas['target']=1
				datas.to_csv('data.csv',mode='a',header=False)
				i=i+8
				j=j+1
				good=datas.columns
			else:
				if np.random.random()>0.3:
					datas=a.iloc[i:i+1]
					datas['code']=code
					datas['target']=0
					print(datas.shape)
					print(datas.columns)
					datas.to_csv('data.csv',mode='a',header=False)
				i=i+1

good=[]

def kdj(data,date,m1,m2):
	data_use=data[['high','low','open','close']]
	data['lown'] = pd.rolling_min(data_use['low'], date)
	data.lown.fillna(value=pd.expanding_min(data_use['low']), inplace=True)
	data['highn'] = pd.rolling_max(data_use['high'], date)
	data.highn.fillna(value=pd.expanding_max(data_use['high']), inplace=True)
	data['rsv']=(data['close'] - data['lown']) / (data['highn'] - data['lown']) * 100
	data['kdj_k'] = pd.ewma(data['rsv'], m1)
	data['kdj_d'] = pd.ewma(data['kdj_k'], m2)
	data['kdj_j'] = 3 * data['kdj_k'] - 2 * data['kdj_d']

def kdj_k_min_10(code):
	a=gc.cleanData(code)
	if a is not None:
		if a.ix[-1:].kdj_k.get_values()<10:
			good.append(code)

def calculateLocation(now,max,min):
	mean=(max+min)/2
	return np.sign(now-mean)*(np.sqrt((now-mean)**2)/mean)+1

needCol=['MA_15','MA_30', 'MA_60', 'MA_90', 'MA_120']
def fiveLine(code):
	try:
		a=gc.getData(code)
	except:
		print('{} is wrong'.format(code))
	else:
		print('{} is running'.format(code))
		if a is not None:
			a.sort_index(inplace=True)
			gc.ma(a,'close',[15,30,60,90,120])
			return a

def calculateFive(a,needCol):
	std=a[needCol].std(axis=1).get_values()
	max=a[needCol].max(axis=1).get_values()
	min=a[needCol].min(axis=1).get_values()
	val_15=a.MA_15.get_values()
	val_now=a.close.get_values()
	loc_15=calculateLocation(a.MA_15,max,min).get_values()
	loc_now=calculateLocation(a.close,a.MA_15,a.MA_15).get_values()
	print('{} , {} , {} , {}'.format(val_15,val_now,loc_now,std))
	if std<0.3 and val_15<max:
		if loc_now>1.02 and val_now<max:
			return True

code=gd.get_all_code()

def useFive(code):
	a=fiveLine(code)
	global can_try
	global test_try
	if a is not None and len(a)>30:
		for i in range(len(a)-10,len(a)-5):
			if calculateFive(a.iloc[i:i+1],needCol):
				can_try+=1
				if a.iloc[i+5:i+6]['close'].get_values()>a.iloc[i:i+1]['close'].get_values():
					test_try+=1
					good.append((a.iloc[i+5:i+6]['close'].get_values()-a.iloc[i:i+1]['close'].get_values())/a.iloc[i:i+1]['close'].get_values())

need=[]
report=[]
def getFive(code):
	a=fiveLine(code)
	trys=0
	tests=0
	tmp=[]
	if a is not None and len(a)>30:
		for i in range(30,len(a)):
			if calculateFive(a.iloc[i:i+1],needCol):
				trys+=1
				tmp.append((a.iloc[i+5:i+6]['close'].get_values()-a.iloc[i:i+1]['close'].get_values())/a.iloc[i:i+1]['close'].get_values())
				good.append((a.iloc[i+5:i+6]['close'].get_values()-a.iloc[i:i+1]['close'].get_values())/a.iloc[i:i+1]['close'].get_values())
				if a.iloc[i+5:i+6]['close'].get_values()>a.iloc[i:i+1]['close'].get_values():
					tests+=1
		if trys==tests:
			need.append(code)
			strs='{} is good, rate is : {}'.format(code,np.mean(tmp))
			report.append(strs)

# for j in code:
# 	kdj_k_min_10(j)

for j in code:
	getFive(j)

for z in report:
	print(z)
print(need)
print(np.mean(good))
# print('total:{},try:{},good:{}'.format(total,can_try,test_try))

# if test_try/can_try>0.5:
# 	x=np.array(diff_list)
# 	print('this strage max:{},min:{},mean:{},std:{}'.format(x.max(),x.min(),x.mean(),x.std()))

# print(good)
# 	get_low(code)

# print(good)

# good=['600234', '600959', '601016', '601021', '603019', '603021', '603028', '603029', '603085', '603117', '603131', '603169', '603223', '603315', '603318', '603338', '603528', '603588', '603598', '603601', '603663', '603678', '603686', '603703', '603779', '603818', '603869', '603959', '000897', '002143', '002195', '002240', '002354', '002520', '002577', '002602', '002681', '002739', '002743', '002746', '002747', '002751', '002752', '002771', '002773', '002777', '002780', '002785', '002796', '002798', '002799', '002800', '002801', '002803', '002805', '002806']

# j=0

# for i in good:
# 	diff=test.test(i,'2016-09-19','2016-09-20')
# 	if diff>0:
# 		j=j+1

# print('total:{},good:{}'.format(len(good),j))		<|endoftext|>


<commit_before>from django.conf.urls import url
from . import views

app_name = 'MyProjects'

urlpatterns = [
   #/MyProjects/detail
    url(r'^detail/', views.detail, name = 'detail'),

   #/MyProjects/index
    url(r'index/', views.index, name = 'index'),

   #/MyProjects/index/jobquery
    url(r'jobquery/', views.jobquery, name = 'jobquery'),

    url(r'jobsearch_result/',views.search_job, name='jobsearch_result'),

]

<commit_msg>Delete urls.py<commit_after><|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

from __future__ import absolute_import

import itertools as it

import pytest

import numpy as np

import scipy.ndimage as spnd

import dask.array as da

import dask_ndmeasure
import dask_ndmeasure._test_utils

@pytest.mark.parametrize(
    "funcname", [
        "center_of_mass",
        "extrema",
        "maximum",
        "maximum_position",
        "mean",
        "median",
        "minimum",
        "minimum_position",
        "standard_deviation",
        "sum",
        "variance",
    ]
)
def test_measure_props_err(funcname):
    da_func = getattr(dask_ndmeasure, funcname)

    shape = (15, 16)
    chunks = (4, 5)
    ind = None

    a = np.random.random(shape)
    d = da.from_array(a, chunks=chunks)

    lbls = (a < 0.5).astype(np.int64)
    d_lbls = da.from_array(lbls, chunks=d.chunks)

    lbls = lbls[:-1]
    d_lbls = d_lbls[:-1]

    with pytest.raises(ValueError):
        da_func(d, lbls, ind)

@pytest.mark.parametrize(
    "funcname", [
        "center_of_mass",
        "maximum",
        "maximum_position",
        "mean",
        "median",
        "minimum",
        "minimum_position",
        "standard_deviation",
        "sum",
        "variance",
    ]
)
@pytest.mark.parametrize(
    "shape, chunks, has_lbls, ind", [
        ((5, 6, 4), (2, 3, 2), False, None),
        ((15, 16), (4, 5), False, None),
        ((15, 16), (4, 5), True, None),
        ((15, 16), (4, 5), True, 0),
        ((15, 16), (4, 5), True, 1),
        ((15, 16), (4, 5), True, [1]),
        ((15, 16), (4, 5), True, [1, 2]),
        ((5, 6, 4), (2, 3, 2), True, [1, 2]),
        ((15, 16), (4, 5), True, [1, 100]),
        ((5, 6, 4), (2, 3, 2), True, [1, 100]),
        ((15, 16), (4, 5), True, [[1, 2, 3, 4]]),
        ((15, 16), (4, 5), True, [[1, 2], [3, 4]]),
        ((15, 16), (4, 5), True, [[[1], [2], [3], [4]]]),
    ]
)
def test_measure_props(funcname, shape, chunks, has_lbls, ind):
    sp_func = getattr(spnd, funcname)
    da_func = getattr(dask_ndmeasure, funcname)

    a = np.random.random(shape)
    d = da.from_array(a, chunks=chunks)

    lbls = None
    d_lbls = None

    if has_lbls:
        lbls = np.zeros(a.shape, dtype=np.int64)
        lbls += (
            (a < 0.5).astype(lbls.dtype) +
            (a < 0.25).astype(lbls.dtype) +
            (a < 0.125).astype(lbls.dtype) +
            (a < 0.0625).astype(lbls.dtype)
        )
        d_lbls = da.from_array(lbls, chunks=d.chunks)

    a_r = np.array(sp_func(a, lbls, ind))
    d_r = da_func(d, d_lbls, ind)

    assert a_r.dtype == d_r.dtype
    assert a_r.shape == d_r.shape

    # See the linked issue for details.
    # ref: https://github.com/scipy/scipy/issues/7706
    if (funcname == "median" and
        ind is not None and
        not np.in1d(np.atleast_1d(ind), lbls).all()):
        pytest.skip("SciPy's `median` mishandles missing labels.")

    assert np.allclose(np.array(a_r), np.array(d_r), equal_nan=True)

@pytest.mark.parametrize(
    "shape, chunks, has_lbls, ind", [
        ((15, 16), (4, 5), False, None),
        ((5, 6, 4), (2, 3, 2), False, None),
        ((15, 16), (4, 5), True, None),
        ((15, 16), (4, 5), True, 0),
        ((15, 16), (4, 5), True, 1),
        ((15, 16), (4, 5), True, [1]),
        ((15, 16), (4, 5), True, [1, 2]),
        ((5, 6, 4), (2, 3, 2), True, [1, 2]),
        ((15, 16), (4, 5), True, [1, 100]),
        ((5, 6, 4), (2, 3, 2), True, [1, 100]),
        ((15, 16), (4, 5), True, [[1, 2, 3, 4]]),
        ((15, 16), (4, 5), True, [[1, 2], [3, 4]]),
        ((15, 16), (4, 5), True, [[[1], [2], [3], [4]]]),
    ]
)
def test_extrema(shape, chunks, has_lbls, ind):
    a = np.random.random(shape)
    d = da.from_array(a, chunks=chunks)

    lbls = None
    d_lbls = None

    if has_lbls:
        lbls = np.zeros(a.shape, dtype=np.int64)
        lbls += (
            (a < 0.5).astype(lbls.dtype) +
            (a < 0.25).astype(lbls.dtype) +
            (a < 0.125).astype(lbls.dtype) +
            (a < 0.0625).astype(lbls.dtype)
        )
        d_lbls = da.from_array(lbls, chunks=d.chunks)

    a_r = spnd.extrema(a, lbls, ind)
    d_r = dask_ndmeasure.extrema(d, d_lbls, ind)

    assert len(a_r) == len(d_r)

    for i in range(len(a_r)):
        a_r_i = np.array(a_r[i])
        assert a_r_i.dtype == d_r[i].dtype
        assert a_r_i.shape == d_r[i].shape
        assert np.allclose(a_r_i, np.array(d_r[i]), equal_nan=True)

@pytest.mark.parametrize(
    "shape, chunks, has_lbls, ind", [
        ((15, 16), (4, 5), False, None),
        ((5, 6, 4), (2, 3, 2), False, None),
        ((15, 16), (4, 5), True, None),
        ((15, 16), (4, 5), True, 0),
        ((15, 16), (4, 5), True, 1),
        ((15, 16), (4, 5), True, 100),
        ((15, 16), (4, 5), True, [1]),
        ((15, 16), (4, 5), True, [1, 2]),
        ((5, 6, 4), (2, 3, 2), True, [1, 2]),
        ((15, 16), (4, 5), True, [1, 100]),
        ((5, 6, 4), (2, 3, 2), True, [1, 100]),
    ]
)
@pytest.mark.parametrize(
    "min, max, bins", [
        (0, 1, 5),
    ]
)
def test_histogram(shape, chunks, has_lbls, ind, min, max, bins):
    a = np.random.random(shape)
    d = da.from_array(a, chunks=chunks)

    lbls = None
    d_lbls = None

    if has_lbls:
        lbls = np.zeros(a.shape, dtype=np.int64)
        lbls += (
            (a < 0.5).astype(lbls.dtype) +
            (a < 0.25).astype(lbls.dtype) +
            (a < 0.125).astype(lbls.dtype) +
            (a < 0.0625).astype(lbls.dtype)
        )
        d_lbls = da.from_array(lbls, chunks=d.chunks)

    a_r = spnd.histogram(a, min, max, bins, lbls, ind)
    d_r = dask_ndmeasure.histogram(d, min, max, bins, d_lbls, ind)

    if ind is None or np.isscalar(ind):
        if a_r is None:
            assert d_r.compute() is None
        else:
            np.allclose(a_r, d_r.compute(), equal_nan=True)
    else:
        assert a_r.dtype == d_r.dtype
        assert a_r.shape == d_r.shape
        for i in it.product(*[range(_) for _ in a_r.shape]):
            if a_r[i] is None:
                assert d_r[i].compute() is None
            else:
                assert np.allclose(a_r[i], d_r[i].compute(), equal_nan=True)

@pytest.mark.parametrize(
    "shape, chunks, ind", [
        ((15, 16), (4, 5), None),
        ((5, 6, 4), (2, 3, 2), None),
        ((15, 16), (4, 5), 0),
        ((15, 16), (4, 5), 1),
        ((15, 16), (4, 5), [1]),
        ((15, 16), (4, 5), [1, 2]),
        ((5, 6, 4), (2, 3, 2), [1, 2]),
        ((15, 16), (4, 5), [1, 100]),
        ((5, 6, 4), (2, 3, 2), [1, 100]),
    ]
)
@pytest.mark.parametrize(
    "default", [
        None,
        0,
        1.5,
    ]
)
@pytest.mark.parametrize(
    "pass_positions", [
        False,
        True,
    ]
)
def test_labeled_comprehension(shape, chunks, ind, default, pass_positions):
    a = np.random.random(shape)
    d = da.from_array(a, chunks=chunks)

    lbls = np.zeros(a.shape, dtype=np.int64)
    lbls += (
        (a < 0.5).astype(lbls.dtype) +
        (a < 0.25).astype(lbls.dtype) +
        (a < 0.125).astype(lbls.dtype) +
        (a < 0.0625).astype(lbls.dtype)
    )
    d_lbls = da.from_array(lbls, chunks=d.chunks)

    def func(val, pos=None):
        if pos is None:
            pos = 0 * val + 1

        return (val * pos).sum() / (1 + val.max() * pos.max())

    a_cm = spnd.labeled_comprehension(
        a, lbls, ind, func, np.float64, default, pass_positions
    )
    d_cm = dask_ndmeasure.labeled_comprehension(
        d, d_lbls, ind, func, np.float64, default, pass_positions
    )

    assert a_cm.dtype == d_cm.dtype
    assert a_cm.shape == d_cm.shape
    assert np.allclose(np.array(a_cm), np.array(d_cm), equal_nan=True)
<commit_msg>Test label<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

from __future__ import absolute_import

import itertools as it

import pytest

import numpy as np

import scipy.ndimage as spnd

import dask.array as da

import dask_ndmeasure
import dask_ndmeasure._test_utils

@pytest.mark.parametrize(
    "funcname", [
        "center_of_mass",
        "extrema",
        "maximum",
        "maximum_position",
        "mean",
        "median",
        "minimum",
        "minimum_position",
        "standard_deviation",
        "sum",
        "variance",
    ]
)
def test_measure_props_err(funcname):
    da_func = getattr(dask_ndmeasure, funcname)

    shape = (15, 16)
    chunks = (4, 5)
    ind = None

    a = np.random.random(shape)
    d = da.from_array(a, chunks=chunks)

    lbls = (a < 0.5).astype(np.int64)
    d_lbls = da.from_array(lbls, chunks=d.chunks)

    lbls = lbls[:-1]
    d_lbls = d_lbls[:-1]

    with pytest.raises(ValueError):
        da_func(d, lbls, ind)

@pytest.mark.parametrize(
    "funcname", [
        "center_of_mass",
        "maximum",
        "maximum_position",
        "mean",
        "median",
        "minimum",
        "minimum_position",
        "standard_deviation",
        "sum",
        "variance",
    ]
)
@pytest.mark.parametrize(
    "shape, chunks, has_lbls, ind", [
        ((5, 6, 4), (2, 3, 2), False, None),
        ((15, 16), (4, 5), False, None),
        ((15, 16), (4, 5), True, None),
        ((15, 16), (4, 5), True, 0),
        ((15, 16), (4, 5), True, 1),
        ((15, 16), (4, 5), True, [1]),
        ((15, 16), (4, 5), True, [1, 2]),
        ((5, 6, 4), (2, 3, 2), True, [1, 2]),
        ((15, 16), (4, 5), True, [1, 100]),
        ((5, 6, 4), (2, 3, 2), True, [1, 100]),
        ((15, 16), (4, 5), True, [[1, 2, 3, 4]]),
        ((15, 16), (4, 5), True, [[1, 2], [3, 4]]),
        ((15, 16), (4, 5), True, [[[1], [2], [3], [4]]]),
    ]
)
def test_measure_props(funcname, shape, chunks, has_lbls, ind):
    sp_func = getattr(spnd, funcname)
    da_func = getattr(dask_ndmeasure, funcname)

    a = np.random.random(shape)
    d = da.from_array(a, chunks=chunks)

    lbls = None
    d_lbls = None

    if has_lbls:
        lbls = np.zeros(a.shape, dtype=np.int64)
        lbls += (
            (a < 0.5).astype(lbls.dtype) +
            (a < 0.25).astype(lbls.dtype) +
            (a < 0.125).astype(lbls.dtype) +
            (a < 0.0625).astype(lbls.dtype)
        )
        d_lbls = da.from_array(lbls, chunks=d.chunks)

    a_r = np.array(sp_func(a, lbls, ind))
    d_r = da_func(d, d_lbls, ind)

    assert a_r.dtype == d_r.dtype
    assert a_r.shape == d_r.shape

    # See the linked issue for details.
    # ref: https://github.com/scipy/scipy/issues/7706
    if (funcname == "median" and
        ind is not None and
        not np.in1d(np.atleast_1d(ind), lbls).all()):
        pytest.skip("SciPy's `median` mishandles missing labels.")

    assert np.allclose(np.array(a_r), np.array(d_r), equal_nan=True)

@pytest.mark.parametrize(
    "shape, chunks, has_lbls, ind", [
        ((15, 16), (4, 5), False, None),
        ((5, 6, 4), (2, 3, 2), False, None),
        ((15, 16), (4, 5), True, None),
        ((15, 16), (4, 5), True, 0),
        ((15, 16), (4, 5), True, 1),
        ((15, 16), (4, 5), True, [1]),
        ((15, 16), (4, 5), True, [1, 2]),
        ((5, 6, 4), (2, 3, 2), True, [1, 2]),
        ((15, 16), (4, 5), True, [1, 100]),
        ((5, 6, 4), (2, 3, 2), True, [1, 100]),
        ((15, 16), (4, 5), True, [[1, 2, 3, 4]]),
        ((15, 16), (4, 5), True, [[1, 2], [3, 4]]),
        ((15, 16), (4, 5), True, [[[1], [2], [3], [4]]]),
    ]
)
def test_extrema(shape, chunks, has_lbls, ind):
    a = np.random.random(shape)
    d = da.from_array(a, chunks=chunks)

    lbls = None
    d_lbls = None

    if has_lbls:
        lbls = np.zeros(a.shape, dtype=np.int64)
        lbls += (
            (a < 0.5).astype(lbls.dtype) +
            (a < 0.25).astype(lbls.dtype) +
            (a < 0.125).astype(lbls.dtype) +
            (a < 0.0625).astype(lbls.dtype)
        )
        d_lbls = da.from_array(lbls, chunks=d.chunks)

    a_r = spnd.extrema(a, lbls, ind)
    d_r = dask_ndmeasure.extrema(d, d_lbls, ind)

    assert len(a_r) == len(d_r)

    for i in range(len(a_r)):
        a_r_i = np.array(a_r[i])
        assert a_r_i.dtype == d_r[i].dtype
        assert a_r_i.shape == d_r[i].shape
        assert np.allclose(a_r_i, np.array(d_r[i]), equal_nan=True)

@pytest.mark.parametrize(
    "shape, chunks, has_lbls, ind", [
        ((15, 16), (4, 5), False, None),
        ((5, 6, 4), (2, 3, 2), False, None),
        ((15, 16), (4, 5), True, None),
        ((15, 16), (4, 5), True, 0),
        ((15, 16), (4, 5), True, 1),
        ((15, 16), (4, 5), True, 100),
        ((15, 16), (4, 5), True, [1]),
        ((15, 16), (4, 5), True, [1, 2]),
        ((5, 6, 4), (2, 3, 2), True, [1, 2]),
        ((15, 16), (4, 5), True, [1, 100]),
        ((5, 6, 4), (2, 3, 2), True, [1, 100]),
    ]
)
@pytest.mark.parametrize(
    "min, max, bins", [
        (0, 1, 5),
    ]
)
def test_histogram(shape, chunks, has_lbls, ind, min, max, bins):
    a = np.random.random(shape)
    d = da.from_array(a, chunks=chunks)

    lbls = None
    d_lbls = None

    if has_lbls:
        lbls = np.zeros(a.shape, dtype=np.int64)
        lbls += (
            (a < 0.5).astype(lbls.dtype) +
            (a < 0.25).astype(lbls.dtype) +
            (a < 0.125).astype(lbls.dtype) +
            (a < 0.0625).astype(lbls.dtype)
        )
        d_lbls = da.from_array(lbls, chunks=d.chunks)

    a_r = spnd.histogram(a, min, max, bins, lbls, ind)
    d_r = dask_ndmeasure.histogram(d, min, max, bins, d_lbls, ind)

    if ind is None or np.isscalar(ind):
        if a_r is None:
            assert d_r.compute() is None
        else:
            np.allclose(a_r, d_r.compute(), equal_nan=True)
    else:
        assert a_r.dtype == d_r.dtype
        assert a_r.shape == d_r.shape
        for i in it.product(*[range(_) for _ in a_r.shape]):
            if a_r[i] is None:
                assert d_r[i].compute() is None
            else:
                assert np.allclose(a_r[i], d_r[i].compute(), equal_nan=True)

@pytest.mark.parametrize(
    "shape, chunks, connectivity", [
        ((15, 16), (4, 5), 1),
        ((15, 16), (15, 16), 1),
        ((15, 16), (15, 16), 2),
        ((5, 6, 4), (5, 6, 4), 1),
        ((5, 6, 4), (5, 6, 4), 2),
        ((5, 6, 4), (5, 6, 4), 3),
    ]
)
def test_label(shape, chunks, connectivity):
    a = np.random.random(shape) < 0.5
    d = da.from_array(a, chunks=chunks)

    s = spnd.generate_binary_structure(a.ndim, connectivity)

    if all([len(c) == 1 for c in d.chunks]):
        a_l, a_nl = spnd.label(a, s)
        d_l, d_nl = dask_ndmeasure.label(d, s)

        assert a_nl == d_nl.compute()

        assert a_l.dtype == d_l.dtype
        assert a_l.shape == d_l.shape
        assert np.allclose(np.array(a_l), np.array(d_l), equal_nan=True)
    else:
        with pytest.raises(ValueError):
            dask_ndmeasure.label(d, s)

@pytest.mark.parametrize(
    "shape, chunks, ind", [
        ((15, 16), (4, 5), None),
        ((5, 6, 4), (2, 3, 2), None),
        ((15, 16), (4, 5), 0),
        ((15, 16), (4, 5), 1),
        ((15, 16), (4, 5), [1]),
        ((15, 16), (4, 5), [1, 2]),
        ((5, 6, 4), (2, 3, 2), [1, 2]),
        ((15, 16), (4, 5), [1, 100]),
        ((5, 6, 4), (2, 3, 2), [1, 100]),
    ]
)
@pytest.mark.parametrize(
    "default", [
        None,
        0,
        1.5,
    ]
)
@pytest.mark.parametrize(
    "pass_positions", [
        False,
        True,
    ]
)
def test_labeled_comprehension(shape, chunks, ind, default, pass_positions):
    a = np.random.random(shape)
    d = da.from_array(a, chunks=chunks)

    lbls = np.zeros(a.shape, dtype=np.int64)
    lbls += (
        (a < 0.5).astype(lbls.dtype) +
        (a < 0.25).astype(lbls.dtype) +
        (a < 0.125).astype(lbls.dtype) +
        (a < 0.0625).astype(lbls.dtype)
    )
    d_lbls = da.from_array(lbls, chunks=d.chunks)

    def func(val, pos=None):
        if pos is None:
            pos = 0 * val + 1

        return (val * pos).sum() / (1 + val.max() * pos.max())

    a_cm = spnd.labeled_comprehension(
        a, lbls, ind, func, np.float64, default, pass_positions
    )
    d_cm = dask_ndmeasure.labeled_comprehension(
        d, d_lbls, ind, func, np.float64, default, pass_positions
    )

    assert a_cm.dtype == d_cm.dtype
    assert a_cm.shape == d_cm.shape
    assert np.allclose(np.array(a_cm), np.array(d_cm), equal_nan=True)
<|endoftext|>


<commit_before>""" helper functions to gather the currency rates """

from urllib.parse import urlencode
import requests
from decimal import Decimal, getcontext

getcontext().prec = 2

def fixer(base, target, value=1, date='latest'):
    """get currency exchange rate from fixer.io in JSON"""

    main_api = 'http://api.fixer.io/{}?'.format(date)
    url = main_api + urlencode({'base':base})
    json_data = requests.get(url).json()

    return round(json_data['rates'][target] * value, 2)

def google_finance_converter(base, target, value):
    """ parse the Google Finance Converter html to extract the value"""
    pass
<commit_msg>refactor fixer function with try/except block<commit_after>""" helper functions to gather the currency rates """

from urllib.parse import urlencode
import requests

def fixer(base, target, value, date='latest'):
    """get currency exchange rate from fixer.io in JSON"""

    main_api = 'http://api.fixer.io/{}?'.format(date)
    url = main_api + urlencode({'base':base})
    json_data = requests.get(url).json()

    try:
        result = round(json_data['rates'][target] * value, 2)
    except requests.exceptions.ConnectionError:
        result = "Connection Error"
    except KeyError:
        result = 1.00 if (base == target) else "Invalid currency"

    return result

#def google_finance_converter(base, target, value):
#    """ parse the Google Finance Converter html to extract the value"""
#    pass
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from openerp.osv import fields, osv
import random
import re
import string
import urllib2
import logging
from openerp import SUPERUSER_ID
from openerp.tools.translate import _
from openerp.tools import html2plaintext
from py_etherpad import EtherpadLiteClient
from openerp.exceptions import UserError

_logger = logging.getLogger(__name__)

class pad_common(osv.osv_memory):
    _name = 'pad.common'

    def pad_is_configured(self, cr, uid, context=None):
        user = self.pool.get('res.users').browse(cr, uid, uid, context=context)
        return bool(user.company_id.pad_server)

    def pad_generate_url(self, cr, uid, context=None):
        company = self.pool.get('res.users').browse(cr, SUPERUSER_ID, uid, context=context).company_id

        pad = {
            "server" : company.pad_server,
            "key" : company.pad_key,
        }

        # make sure pad server in the form of http://hostname
        if not pad["server"]:
            return pad
        if not pad["server"].startswith('http'):
            pad["server"] = 'http://' + pad["server"]
        pad["server"] = pad["server"].rstrip('/')
        # generate a salt
        s = string.ascii_uppercase + string.digits
        salt = ''.join([s[random.SystemRandom().randint(0, len(s) - 1)] for i in range(10)])
        #path
        # etherpad hardcodes pad id length limit to 50
        path = '-%s-%s' % (self._name, salt)
        path = '%s%s' % (cr.dbname.replace('_','-')[0:50 - len(path)], path)
        # contruct the url
        url = '%s/p/%s' % (pad["server"], path)

        #if create with content
        if "field_name" in context and "model" in context and "object_id" in context:
            myPad = EtherpadLiteClient( pad["key"], pad["server"]+'/api')
            try:
                myPad.createPad(path)
            except urllib2.URLError:
                raise UserError(_("Pad creation failed, either there is a problem with your pad server URL or with your connection."))

            #get attr on the field model
            model = self.pool[context["model"]]
            field = model._fields[context['field_name']]
            real_field = field.pad_content_field

            #get content of the real field
            for record in model.browse(cr, uid, [context["object_id"]]):
                if record[real_field]:
                    myPad.setText(path, (html2plaintext(record[real_field]).encode('utf-8')))
                    #Etherpad for html not functional
                    #myPad.setHTML(path, record[real_field])

        return {
            "server": pad["server"],
            "path": path,
            "url": url,
        }

    def pad_get_content(self, cr, uid, url, context=None):
        content = ''
        if url:
            try:
                page = urllib2.urlopen('%s/export/html'%url).read()
                mo = re.search('(.*)',page)
                if mo:
                    content = mo.group(1)
            except:
                _logger.warning("No url found '%s'.", url)
        return content

    # TODO
    # reverse engineer protocol to be setHtml without using the api key

    def write(self, cr, uid, ids, vals, context=None):
        self._set_pad_value(cr, uid, vals, context)
        return super(pad_common, self).write(cr, uid, ids, vals, context=context)

    def create(self, cr, uid, vals, context=None):
        self._set_pad_value(cr, uid, vals, context)
        res = super(pad_common, self).create(cr, uid, vals, context=context)

        # In case the pad is created programmatically, the content is not filled in yet since it is
        # normally initialized by the JS layer
        pad_urls = {}
        for k, field in self._fields.iteritems():
            if hasattr(field, 'pad_content_field') and k not in vals:
                ctx = dict(context or {})
                ctx.update({
                    'model': self._name,
                    'field_name': k,
                    'object_id': res,
                })
                pad = self.pad_generate_url(cr, uid, ctx)
                pad_urls[k] = pad.get('url')
        if pad_urls:
            self.write(cr, uid, res, pad_urls, context=context)
        return res

    # Set the pad content in vals
    def _set_pad_value(self, cr, uid, vals, context=None):
        for k,v in vals.items():
            field = self._fields[k]
            if hasattr(field,'pad_content_field'):
                vals[field.pad_content_field] = self.pad_get_content(cr, uid, v, context=context)

    def copy(self, cr, uid, id, default=None, context=None):
        if not default:
            default = {}
        for k, field in self._fields.iteritems():
            if hasattr(field,'pad_content_field'):
                pad = self.pad_generate_url(cr, uid, context)
                default[k] = pad.get('url')
        return super(pad_common, self).copy(cr, uid, id, default, context)
<commit_msg>[FIX] pad: get pad content with newlines<commit_after># -*- coding: utf-8 -*-
from openerp.osv import fields, osv
import random
import re
import string
import urllib2
import logging
from openerp import SUPERUSER_ID
from openerp.tools.translate import _
from openerp.tools import html2plaintext
from py_etherpad import EtherpadLiteClient
from openerp.exceptions import UserError

_logger = logging.getLogger(__name__)

class pad_common(osv.osv_memory):
    _name = 'pad.common'

    def pad_is_configured(self, cr, uid, context=None):
        user = self.pool.get('res.users').browse(cr, uid, uid, context=context)
        return bool(user.company_id.pad_server)

    def pad_generate_url(self, cr, uid, context=None):
        company = self.pool.get('res.users').browse(cr, SUPERUSER_ID, uid, context=context).company_id

        pad = {
            "server" : company.pad_server,
            "key" : company.pad_key,
        }

        # make sure pad server in the form of http://hostname
        if not pad["server"]:
            return pad
        if not pad["server"].startswith('http'):
            pad["server"] = 'http://' + pad["server"]
        pad["server"] = pad["server"].rstrip('/')
        # generate a salt
        s = string.ascii_uppercase + string.digits
        salt = ''.join([s[random.SystemRandom().randint(0, len(s) - 1)] for i in range(10)])
        #path
        # etherpad hardcodes pad id length limit to 50
        path = '-%s-%s' % (self._name, salt)
        path = '%s%s' % (cr.dbname.replace('_','-')[0:50 - len(path)], path)
        # contruct the url
        url = '%s/p/%s' % (pad["server"], path)

        #if create with content
        if "field_name" in context and "model" in context and "object_id" in context:
            myPad = EtherpadLiteClient( pad["key"], pad["server"]+'/api')
            try:
                myPad.createPad(path)
            except urllib2.URLError:
                raise UserError(_("Pad creation failed, either there is a problem with your pad server URL or with your connection."))

            #get attr on the field model
            model = self.pool[context["model"]]
            field = model._fields[context['field_name']]
            real_field = field.pad_content_field

            #get content of the real field
            for record in model.browse(cr, uid, [context["object_id"]]):
                if record[real_field]:
                    myPad.setText(path, (html2plaintext(record[real_field]).encode('utf-8')))
                    #Etherpad for html not functional
                    #myPad.setHTML(path, record[real_field])

        return {
            "server": pad["server"],
            "path": path,
            "url": url,
        }

    def pad_get_content(self, cr, uid, url, context=None):
        content = ''
        if url:
            try:
                page = urllib2.urlopen('%s/export/html'%url).read()
                mo = re.search('(.*)', page, re.DOTALL)
                if mo:
                    content = mo.group(1)
            except:
                _logger.warning("No url found '%s'.", url)
        return content

    # TODO
    # reverse engineer protocol to be setHtml without using the api key

    def write(self, cr, uid, ids, vals, context=None):
        self._set_pad_value(cr, uid, vals, context)
        return super(pad_common, self).write(cr, uid, ids, vals, context=context)

    def create(self, cr, uid, vals, context=None):
        self._set_pad_value(cr, uid, vals, context)
        res = super(pad_common, self).create(cr, uid, vals, context=context)

        # In case the pad is created programmatically, the content is not filled in yet since it is
        # normally initialized by the JS layer
        pad_urls = {}
        for k, field in self._fields.iteritems():
            if hasattr(field, 'pad_content_field') and k not in vals:
                ctx = dict(context or {})
                ctx.update({
                    'model': self._name,
                    'field_name': k,
                    'object_id': res,
                })
                pad = self.pad_generate_url(cr, uid, ctx)
                pad_urls[k] = pad.get('url')
        if pad_urls:
            self.write(cr, uid, res, pad_urls, context=context)
        return res

    # Set the pad content in vals
    def _set_pad_value(self, cr, uid, vals, context=None):
        for k,v in vals.items():
            field = self._fields[k]
            if hasattr(field,'pad_content_field'):
                vals[field.pad_content_field] = self.pad_get_content(cr, uid, v, context=context)

    def copy(self, cr, uid, id, default=None, context=None):
        if not default:
            default = {}
        for k, field in self._fields.iteritems():
            if hasattr(field,'pad_content_field'):
                pad = self.pad_generate_url(cr, uid, context)
                default[k] = pad.get('url')
        return super(pad_common, self).copy(cr, uid, id, default, context)
<|endoftext|>


<commit_before>from __future__ import unicode_literals

import logging

from django.db import migrations
from osf.utils.migrations import ensure_schemas, remove_schemas

logger = logging.getLogger(__file__)

class Migration(migrations.Migration):

    dependencies = [
        ('osf', '0077_add_maintenance_permissions'),
    ]

    operations = [
        migrations.RunPython(ensure_schemas, remove_schemas),
    ]<commit_msg>fix flake 8<commit_after>from __future__ import unicode_literals

import logging

from django.db import migrations
from osf.utils.migrations import ensure_schemas, remove_schemas

logger = logging.getLogger(__file__)

class Migration(migrations.Migration):

    dependencies = [
        ('osf', '0077_add_maintenance_permissions'),
    ]

    operations = [
        migrations.RunPython(ensure_schemas, remove_schemas),
    ]
<|endoftext|>


<commit_before>import pygame, sys
from reversiboard import *
from reversimoves import *
import random
import time

# Window parameters
WINDOW_WIDTH = 1200
WINDOW_HEIGHT = 900

# Colors
DARK_GREEN = (0, 128, 0)
DARK_GREY = (128, 128, 128)
LIGHT_RED = (255, 192, 192)
GREEN = (0, 255, 0)
WHITE = (255, 255, 255)
BLACK = (0, 0, 0)

# Board size (number of squares on each side)
BOARD_SIZE = 8

HUMAN = 'human'
COMPUTER = 'computer'

# Players: computer is 'W', human is 'B'
# Pick random starting player
sides = [ HUMAN, COMPUTER ]
colors = { HUMAN : WHITE , COMPUTER : BLACK }

another_game = True

while another_game:

    playerIndex = random.randrange(2)
    board = ReversiBoard(BOARD_SIZE, sides)

    pygame.init()
    surface = pygame.display.set_mode((WINDOW_WIDTH, WINDOW_HEIGHT), 0, 32)

    drawer = ReversiBoardDrawer(board,
                                surface,
                                WINDOW_WIDTH,
                                WINDOW_HEIGHT,
                                DARK_GREY,
                                DARK_GREEN,
                                GREEN,
                                sides, colors)

    try:
        playing = True
        missedMoves = 0
        winner = None

        while playing:
            opponentIndex = 1 - playerIndex
            player = sides[playerIndex]
            opponent = sides[opponentIndex]

            drawer.drawBoard()

            moveResult = []

            if board.noLegalMoves(player, opponent):
                print(player + " has no legal move.")
                move = None
                time.sleep(3)
            elif player == HUMAN:
                while moveResult == []:
                    move = getPlayerMove(drawer)
                    moveResult = board.resultOfMove(move, player, opponent)
            else:
                move = getComputerMove(board, COMPUTER, HUMAN)
                moveResult = board.resultOfMove(move, player, opponent)

            if move is None:
                missedMoves += 1

            if missedMoves == 2:
                winner = board.determineWinner()

            board.apply(move, moveResult, player)
            drawer.drawMove(move, player)

            if board.isFull():
                winner = board.determineWinner()
                playing = False

            playerIndex = 1 - playerIndex

    except PlayerQuitException:
        pass

    if winner is None:
        outcome = "The game is a tie."
    else:
        outcome = "The " + winner + " wins!"

    fontObj = pygame.font.Font('freesansbold.ttf', 32)
    textSurface = fontObj.render(outcome, True, LIGHT_RED, DARK_GREY)
    textRect = textSurface.get_rect()
    textRect.center = (WINDOW_WIDTH // 2, WINDOW_HEIGHT // 2)

    surface.blit(textSurface, textRect)
    pygame.display.update()

    asking_about_another_game = True

    while asking_about_another_game:
        for event in pygame.event.get():
            if event.type == QUIT:
                another_game = False
                asking_about_another_game = False
                break
            elif event.type == KEYUP and event.key in [K_ESCAPE, ord('r')]:
                asking_about_another_game = False
                break

        pygame.display.update()

pygame.quit()
sys.exit()
<commit_msg>tweaked loop structure, probably still needs more<commit_after>import pygame, sys
from reversiboard import *
from reversimoves import *
import random
import time

# Window parameters
WINDOW_WIDTH = 1200
WINDOW_HEIGHT = 900

# Colors
DARK_GREEN = (0, 128, 0)
DARK_GREY = (128, 128, 128)
LIGHT_RED = (255, 192, 192)
GREEN = (0, 255, 0)
WHITE = (255, 255, 255)
BLACK = (0, 0, 0)

# Board size (number of squares on each side)
BOARD_SIZE = 8

HUMAN = 'human'
COMPUTER = 'computer'

# Players: computer is 'W', human is 'B'
# Pick random starting player
sides = [ HUMAN, COMPUTER ]
colors = { HUMAN : WHITE , COMPUTER : BLACK }

pygame.init()
surface = pygame.display.set_mode((WINDOW_WIDTH, WINDOW_HEIGHT), 0, 32)

another_game = True

while another_game:

    playerIndex = random.randrange(2)
    board = ReversiBoard(BOARD_SIZE, sides)

    drawer = ReversiBoardDrawer(board,
                                surface,
                                WINDOW_WIDTH,
                                WINDOW_HEIGHT,
                                DARK_GREY,
                                DARK_GREEN,
                                GREEN,
                                sides, colors)

    try:
        playing = True
        missedMoves = 0
        winner = None

        while playing:
            opponentIndex = 1 - playerIndex
            player = sides[playerIndex]
            opponent = sides[opponentIndex]

            drawer.drawBoard()

            moveResult = []

            if board.noLegalMoves(player, opponent):
                print(player + " has no legal move.")
                move = None
                time.sleep(3)
            elif player == HUMAN:
                while moveResult == []:
                    move = getPlayerMove(drawer)
                    moveResult = board.resultOfMove(move, player, opponent)
            else:
                move = getComputerMove(board, COMPUTER, HUMAN)
                moveResult = board.resultOfMove(move, player, opponent)

            if move is None:
                missedMoves += 1
            else:
                missedMoves = 0

            if missedMoves == 2:
                winner = board.determineWinner()
                playing = False
            else:
                board.apply(move, moveResult, player)
                drawer.drawMove(move, player)

                if board.isFull():
                    winner = board.determineWinner()
                    playing = False

                playerIndex = 1 - playerIndex

    except PlayerQuitException:
        pass

    if winner is None:
        outcome = "The game is a tie."
    else:
        outcome = "The " + winner + " wins!"

    fontObj = pygame.font.Font('freesansbold.ttf', 32)
    textSurface = fontObj.render(outcome, True, LIGHT_RED, DARK_GREY)
    textRect = textSurface.get_rect()
    textRect.center = (WINDOW_WIDTH // 2, WINDOW_HEIGHT // 2)

    surface.blit(textSurface, textRect)
    pygame.display.update()

    asking_about_another_game = True

    while asking_about_another_game:
        for event in pygame.event.get():
            if event.type == QUIT:
                another_game = False
                asking_about_another_game = False
                break
            elif event.type == KEYUP and event.key in [K_ESCAPE, ord('r')]:
                asking_about_another_game = False
                break

        pygame.display.update()

pygame.quit()
sys.exit()
<|endoftext|>


<commit_before>from django.db import models

class Module(models.Model):
    """
    "modules" => "`moduleid` int(10) unsigned NOT NULL auto_increment,
                `file` varchar(45) NOT NULL default '',
                `ordernum` int(10) unsigned NOT NULL default '0',
                `description` varchar(45) NOT NULL default '',
                `required` varchar(45) default NULL,
                PRIMARY KEY  (`moduleid`)",
    """

    # `file` varchar(45) NOT NULL default '',

    # in this implementation, we store the name of a
    # template tag instead
    name = models.CharField(max_length=45)

    # `ordernum` int(10) unsigned NOT NULL default '0',
    ordernum = models.IntegerField()

    enabled = models.BooleanField(default=True)

    # `description` varchar(45) NOT NULL default '',
    description = models.CharField(max_length=45)

    # `required` varchar(45) default NULL,
    required = models.CharField(max_length=45)

    def __str__(self):
        return '{} {} at position {}'.format(
            'enabled' if self.enabled else 'disabled',
            self.name,
            self.ordernum
        )
<commit_msg>Let required be empty<commit_after>from django.db import models

class Module(models.Model):
    """
    "modules" => "`moduleid` int(10) unsigned NOT NULL auto_increment,
                `file` varchar(45) NOT NULL default '',
                `ordernum` int(10) unsigned NOT NULL default '0',
                `description` varchar(45) NOT NULL default '',
                `required` varchar(45) default NULL,
                PRIMARY KEY  (`moduleid`)",
    """

    # `file` varchar(45) NOT NULL default '',

    # in this implementation, we store the name of a
    # template tag instead
    name = models.CharField(max_length=45)

    # `ordernum` int(10) unsigned NOT NULL default '0',
    ordernum = models.IntegerField()

    enabled = models.BooleanField(default=True)

    # `description` varchar(45) NOT NULL default '',
    description = models.CharField(max_length=45)

    # `required` varchar(45) default NULL,
    required = models.CharField(max_length=45, null=True, blank=True)

    def __str__(self):
        return '{} {} at position {}'.format(
            'enabled' if self.enabled else 'disabled',
            self.name,
            self.ordernum
        )
<|endoftext|>


<commit_before>#Tim's solution to deg.py

from math import pi
for deg in range(0,181,10):
    rad = deg*pi / 180.
    print deg, rad

<commit_msg>changed comment<commit_after>"""
Tim's solution to deg.py
"""

from math import pi
for deg in range(0,181,10):
    rad = deg*pi / 180.
    print deg, rad

<|endoftext|>


<commit_before>from math import tanh
from sqlite3 import dbapi2 as sqlite

class searchnet:
    def __init__(self, dbname):
        self.con = sqlite.connect(dbname)

    def __del__(self):
        self.con.close()

    def maketables(self):
        self.con.execute('create table hiddennode(create_key)')
        self.con.execute('create table inhidden(fromid, toid, strength)')
        self.con.execute('create table hiddenout(fromid, toid, strength)')
        self.con.commit()

    def getstrength(self, fromid, toid, layer):
        if layer == 0:
            table = 'inhidden'
        else:
            table = 'hiddenout'
        res = self.con.execute(
                'select strength from %s where formid = %d and toid = %d'
                % (table, fromid, toid)).fetchone()
        if res == None:
            if layer == 0:
                return -0.2
            if layer == 1:
                return 0
        return res[0]

    def setstrength(self, fromid, toid, layer, strength):
        if layer == 0:
            table = 'inhidden'
        else:
            table = 'hiddenout'
        res = self.con.execute(
                'select rowid from %s where fromid = %d and toid = %d'
                % (table, fromid, toid)).fetchone()
        if res == None:
            self.con.execute(
                    'insert into %s (fromid, toid, strength) values (?, ?, ?'
                    % (table, fromid, toid, strength))
        else:
            rowid = res[0]
            self.con.execute(
                    'update %s set strength = %f where rowid = %d'
                    % (table, strength, rowid))

    def generatehiddennode(self, inputs, outputs):
        if len(inputs) > 3:
            return None
        createkey = '_'.join(sorted([str(it) for it in inputs]))
        res = self.con.execute(
                "select rowid from hiddennode where create_key = '%s'"
                % createkey).fetchone()

        if res == None:
            cur = self.con.execute(
                    "insert into hiddennode (create_key) values ('%s')"
                    % createkey)
            hiddenid = cur.lastrowid
            for ip in inputs:
                self.setstrength(ip, hiddenid, 0, 1.0 / len(inputs))
            for op in outputs:
                self.setstrength(hiddenid, op, 1, 0.1)
            self.con.commit()
<commit_msg>add train network<commit_after>from math import tanh
from sqlite3 import dbapi2 as sqlite

def dtanh(y):
    return 1.0 - y * y

class searchnet:
    def __init__(self, dbname):
        self.con = sqlite.connect(dbname)

    def __del__(self):
        self.con.close()

    def maketables(self):
        self.con.execute('create table hiddennode(create_key)')
        self.con.execute('create table inhidden(fromid, toid, strength)')
        self.con.execute('create table hiddenout(fromid, toid, strength)')
        self.con.commit()

    def getstrength(self, fromid, toid, layer):
        if layer == 0:
            table = 'inhidden'
        else:
            table = 'hiddenout'
        res = self.con.execute(
                'select strength from %s where fromid = %d and toid = %d'
                % (table, fromid, toid)).fetchone()
        if res == None:
            if layer == 0:
                return -0.2
            if layer == 1:
                return 0
        return res[0]

    def setstrength(self, fromid, toid, layer, strength):
        if layer == 0:
            table = 'inhidden'
        else:
            table = 'hiddenout'
        res = self.con.execute(
                'select rowid from %s where fromid = %d and toid = %d'
                % (table, fromid, toid)).fetchone()
        if res == None:
            self.con.execute(
                    'insert into %s (fromid, toid, strength) values (%d, %d, %f)'
                    % (table, fromid, toid, strength))
        else:
            rowid = res[0]
            self.con.execute(
                    'update %s set strength = %f where rowid = %d'
                    % (table, strength, rowid))

    def generatehiddennode(self, inputids, outputids):
        if len(inputids) > 3:
            return None
        createkey = '_'.join(sorted([str(ii) for ii in inputids]))
        res = self.con.execute(
                "select rowid from hiddennode where create_key = '%s'"
                % createkey).fetchone()

        if res == None:
            cur = self.con.execute(
                    "insert into hiddennode (create_key) values ('%s')"
                    % createkey)
            hiddenid = cur.lastrowid
            for inputid in inputids:
                self.setstrength(inputid, hiddenid, 0, 1.0 / len(inputids))
            for outputid in outputids:
                self.setstrength(hiddenid, outputid, 1, 0.1)
            self.con.commit()

    def getallhiddenids(self, inputids, outputids):
        l1 = {}
        for inputid in inputids:
            cur = self.con.execute(
                    'select toid from inhidden where fromid = %d'
                    % inputid)
            for row in cur:
                l1[row[0]] = 1
        for outputid in outputids:
            cur = self.con.execute(
                    'select fromid from hiddenout where toid = %d'
                    % outputid)
            for row in cur:
                l1[row[0]] = 1
        return l1.keys()

    def setupnetwork(self, inputids, outputids):
        self.inputids = inputids
        self.hiddenids = self.getallhiddenids(inputids, outputids)
        self.outputids = outputids

        self.ai = [1.0] * len(self.inputids)
        self.ah = [1.0] * len(self.hiddenids)
        self.ao = [1.0] * len(self.outputids)

        self.wi = [[self.getstrength(inputid, hiddenid, 0)
            for hiddenid in self.hiddenids]
            for inputid in self.inputids]
        self.wo = [[self.getstrength(hiddenid, outputid, 1)
            for outputid in self.outputids]
            for hiddenid in self.hiddenids]

    def feedforward(self):
        for i in range(len(self.inputids)):
            self.ai[i] = 1.0

        for j in range(len(self.hiddenids)):
            sum = 0.0
            for i in range(len(self.inputids)):
                sum = sum + self.ai[i] * self.wi[i][j]
            self.ah[j] = tanh(sum)

        for k in range(len(self.outputids)):
            sum = 0.0
            for j in range(len(self.hiddenids)):
                sum = sum + self.ah[j] * self.wo[j][k]
            self.ao[k] = tanh(sum)
        return self.ao[:]

    def getresult(self, inputids, outputids):
        self.setupnetwork(inputids, outputids)
        return self.feedforward()

    def backpropagate(self, targets, N = 0.5):
        outputdeltas = [0.0] * len(self.outputids)
        for k in range(len(self.outputids)):
            error = targets[k] - self.ao[k]
            outputdeltas[k] = dtanh(self.ao[k]) * error

        hiddendeltas = [0.0] * len(self.hiddenids)
        for j in range(len(self.hiddenids)):
            error = 0.0
            for k in range(len(self.outputids)):
                error += outputdeltas[k] * self.wo[j][k]
            hiddendeltas[j] = dtanh(self.ah[j]) * error

        for j in range(len(self.hiddenids)):
            for k in range(len(self.outputids)):
                change = outputdeltas[k] * self.ah[j]
                self.wo[j][k] += N * change

        for i in range(len(self.inputids)):
            for j in range(len(self.hiddenids)):
                change = hiddendeltas[j] * self.ai[i]
                self.wi[i][j] += N * change

    def trainquery(self, inputids, outputids, selectedoutput):
        self.generatehiddennode(inputids, outputids)
        self.setupnetwork(inputids, outputids)
        self.feedforward()
        targets = [0.0] * len(outputids)
        targets[outputids.index(selectedoutput)] = 1.0
        error = self.backpropagate(targets)
        self.updatedatabase()

    def updatedatabase(self):
        for i in range(len(self.inputids)):
            for j in range(len(self.hiddenids)):
                self.setstrength(self.inputids[i], self.hiddenids[j],
                        0, self.wi[i][j])
        for j in range(len(self.hiddenids)):
            for k in range(len(self.outputids)):
                self.setstrength(self.hiddenids[j], self.outputids[k],
                        1, self.wo[j][k])
        self.con.commit()
<|endoftext|>