<commit_before>#
# Copyright (c) 2004 Specifix, Inc.
# All rights reserved
#
import imp, sys
import os
import util
import build
import package
import shutil
import types
import inspect
import lookaside
import rpmhelper
import gzip

baseMacros = (
    # Note that these macros cannot be represented as a dictionary,
    # because the items need to be added in order so that they will
    # be properly interpolated.
    #
    # paths
    ('prefix'		, '/usr'),
    ('sysconfdir'	, '/etc'),
    ('lib'              , 'lib'),  # may be overridden with 'lib64'
    ('exec_prefix'	, '%(prefix)s'),
    ('bindir'		, '%(exec_prefix)s/bin'),
    ('sbindir'		, '%(exec_prefix)s/sbin'),
    ('libdir'		, '%(exec_prefix)s/%(lib)s'),
    ('libexecdir'	, '%(exec_prefix)s/libexec'),
    ('localstatedir'	, '%(prefix)s/var'),
    ('sharedstatedir'	, '%(prefix)s/com'),
    ('includedir'	, '%(prefix)s/include'),
    ('datadir'		, '%(prefix)s/share'),
    ('mandir'		, '%(datadir)s/man'),
    ('infodir'		, '%(datadir)s/info'),
    ('docdir'		, '%(datadir)s/doc'),
    ('develdocdir'	, '%(datadir)s/develdoc'),
    # arguments/flags (empty ones are for documentation; non-existant = empty)
    ('cflags'           , '-O2'),
    ('mflags'		, ''),
    ('parallelmflags'   , ''),
    ('sysroot'		, ''),
)

crossMacros = (
    # set crossdir from cook, directly or indirectly, before adding the rest
    #('crossdir'	, 'cross-target'),
    ('prefix'		, '/opt/%(crossdir)s'),
    ('sysroot'		, '%(prefix)s/sys-root'),
    ('headerpath'	, '%(sysroot)s/usr/include')
)

# XXX TEMPORARY - remove directories such as /usr/include from this
# list when filesystem package is in place.
baseAutoSpec = (
    # automatic subpackage names and sets of regexps that define them
    # cannot be a dictionary because it is ordered; first match wins
    ('devel',
	('\.a',
	 '\.so',
	 '.*/include/.*\.h',
	 '/usr/include/',
         '/usr/include',
	 '/usr/share/man/man(2|3)/',
	 '/usr/share/man/man(2|3)',
	 '/usr/share/develdoc/',
	 '/usr/share/develdoc',
	 '/usr/share/aclocal/',
         '/usr/share/aclocal')),
    ('lib', ('.*/lib/.*\.so\..*')),
    ('doc', ('/usr/share/(doc|man|info)/',
             '/usr/share/(doc|man|info)')),
    ('locale', ('/usr/share/locale/',
                '/usr/share/locale')),
    ('runtime', ('.*',)),
)

class Macros(dict):
    def __setitem__(self, name, value):
	dict.__setitem__(self, name, value % self)

    # we want keys that don't exist to default to empty strings
    def __getitem__(self, name):
	if self.has_key(name):
	    return dict.__getitem__(self, name)
	return ''

    def addMacros(self, *macroSet):
	# must be in order; later macros in the set can depend on
	# earlier ones
	# for ease of use, we allow passing in a tuple of tuples, or
	# a simple set of tuples
	if len(macroSet) == 1 and type(macroSet[0]) is tuple:
	    # we were passed a tuple of tuples (like baseMacros)
	    macroSet = macroSet[0]
        if len(macroSet) > 0 and type(macroSet[0]) is not tuple:
            # we were passed something like ('foo', 'bar')
            macroSet = (macroSet,)
	for key, value in macroSet:
	    self[key] = value

    def copy(self):
	new = Macros()
	new.update(self)
	return new

def flatten(list):
    if type(list) != types.ListType: return [list]
    if list == []: return list
    return flatten(list[0]) + flatten(list[1:])

def extractSourceFromRPM(rpm, targetfile):
    filename = os.path.basename(targetfile)
    directory = os.path.dirname(targetfile)
    r = file(rpm, 'r')
    rpmhelper.seekToData(r)
    gz = gzip.GzipFile(fileobj=r)
    (rpipe, wpipe) = os.pipe()
    pid = os.fork()
    if not pid:
	os.dup2(rpipe, 0)
	os.chdir(directory)
	os.execl('/bin/cpio', 'cpio', '-ium', filename)
	sys.exit(1)
    while 1:
	buf = gz.read(4096)
	if not buf:
	    break
	os.write(wpipe, buf)
    os.close(wpipe)
    (pid, status) = os.waitpid(pid, 0)
    if not os.WIFEXITED(status):
	raise IOError, 'cpio died extracting %s from RPM %s' \
	               %(filename, os.path.basename(rpm))
    if os.WEXITSTATUS(status):
	raise IOError, 'cpio returned failure %d extracting %s from RPM %s' \
	               %(os.WEXITSTATUS(status), filename, os.path.basename(rpm))
    if not os.path.exists(targetfile):
	raise IOError, 'failed to extract source %s from RPM %s' \
	               %(filename, os.path.basename(rpm))

class RecipeLoader(types.DictionaryType):
    def __init__(self, file):
        if file[0] != "/":
            raise IOError, "recipe file names must be absolute paths"

        self.file = os.path.basename(file).replace('.', '-')
        self.module = imp.new_module(self.file)
        sys.modules[self.file] = self.module
        f = open(file)

        exec 'from recipe import Recipe' in self.module.__dict__
        exec 'from recipe import loadRecipe' in self.module.__dict__
        exec 'import build, os, package, sys, util' in self.module.__dict__
        if sys.excepthook == util.excepthook:
            exec 'sys.excepthook = util.excepthook' in self.module.__dict__
        exec 'filename = "%s"' %(file) in self.module.__dict__
        code = compile(f.read(), file, 'exec')
        exec code in self.module.__dict__
        for (name, obj) in  self.module.__dict__.items():
            if type(obj) == types.ClassType:
                # make sure the class is derived from Recipe
                # and has a name
                if obj.__dict__.has_key('ignore'):
                    continue
                if issubclass(obj, Recipe) and obj.__dict__.has_key('name'):
                    obj.__dict__['filename'] = file
                    self[name] = obj

    def __del__(self):
        try:
            del sys.modules[self.file]
        except:
            pass

# XXX this should be extended to load a recipe from srs
def loadRecipe(file):
    callerGlobals = inspect.stack()[1][0].f_globals
    if file[0] != '/':
        recipepath = os.path.dirname(callerGlobals['filename'])
        file = recipepath + '/' + file
    recipes = RecipeLoader(file)
    for name, recipe in recipes.items():
        # XXX hack to hide parent recipes
        recipe.ignore = 1
        callerGlobals[name] = recipe
        # stash a reference to the module in the namespace
        # of the recipe that loaded it, or else it will be destroyed
        callerGlobals[os.path.basename(file).replace('.', '-')] = recipes

#def bootstrapRecipe(file, class, buildRequires):
#    loadRecipe(file) # XXX not necessary if we put boostraps in main files
#    exec """class Bootstrap%s(%s):
#	buildRequires = %s
#	name = "bootstrap-%s"
#	def setup(self):
#	    FIXMEcrossmacros(self.recipeCfg)
#	    FIXMEcrossenv
#	    FIXMEself.mainDir(class, self.version)
#	    %s.setup(self)
#    """ %(class, class, buildRequires.repr(), class, class)

class Recipe:
    buildRequires = []
    runRequires = []

    def addSignature(self, file, keyid):
	# do not search unless a gpg keyid is specified
	if not keyid:
	    return
	gpg = '%s.sig' %(file)
	c = lookaside.searchAll(self.cfg, self.laReposCache, gpg,
				self.name, self.srcdirs)
	if not c:
	    gpg = '%s.sign' %(file)
	    c = lookaside.searchAll(self.cfg, self.laReposCache,
				    gpg, self.name, self.srcdirs)
	if c:
	    if not self.signatures.has_key(file):
		self.signatures[file] = []
	    self.signatures[file].append((gpg, c, keyid))

    def addTarball(self, file, extractDir='', keyid=None):
	self.tarballs.append((file, extractDir))
	self.addSignature(file, keyid)

    def addTarballFromRPM(self, rpm, file, extractDir='', keyid=None):
	f = lookaside.searchAll(self.cfg, self.laReposCache,
			     os.path.basename(file), self.name, self.srcdirs)
	if not f:
	    r = lookaside.findAll(self.cfg, self.laReposCache, rpm,
				  self.name, self.srcdirs)
	    c = lookaside.createCacheName(self.cfg, file, self.name)
	    extractSourceFromRPM(r, c)
	    f = lookaside.findAll(self.cfg, self.laReposCache, file,
				  self.name, self.srcdirs)
	self.tarballs.append((file, extractDir))
	self.addSignature(f, keyid)

    def addPatch(self, file, level='1', backup='', keyid=None):
	self.patches.append((file, level, backup))
	self.addSignature(file, keyid)

    def addSource(self, file, keyid=None):
	self.sources.append(file)
	self.addSignature(file, keyid)

    def allSources(self):
        sources = []
        for (tarball, extractdir) in self.tarballs:
            sources.append(tarball)
        for (patch, level, backup) in self.patches:
            sources.append(patch)
	for signaturelist in self.signatures.values():
            for (gpg, cached, keyid) in signaturelist:
                sources.append(gpg)
	return sources + self.sources

    def mainDir(self, new = None):
	if new:
	    self.theMainDir = new

	return self.theMainDir

    def nameVer(self):
	return self.name + "-" + self.version

    def cleanup(self, builddir, destdir):
	shutil.rmtree(builddir)
	shutil.rmtree(destdir)

    def checkSignatures(self, filepath, file):
        if not self.signatures.has_key(file):
            return
	for (gpg, signature, keyid) in self.signatures[file]:
	    # FIXME: our own keyring
	    if os.system("gpg --no-secmem-warning --verify %s %s"
			  %(signature, filepath)):
		# FIXME: only do this if key missing, this is cheap for now
		os.system("gpg --keyserver pgp.mit.edu --recv-keys 0x %s"
		          %(keyid))
		if os.system("gpg --no-secmem-warning --verify %s %s"
			      %(signature, filepath)):
		    raise RuntimeError, "GPG signature %s failed" %(signature)

    def unpackSources(self, builddir):
	if os.path.exists(builddir):
	    shutil.rmtree(builddir)
	util.mkdirChain(builddir)
	for (file, extractdir) in self.tarballs:
            f = lookaside.findAll(self.cfg, self.laReposCache, file,
				  self.name, self.srcdirs)
	    self.checkSignatures(f, file)
            if f.endswith(".bz2"):
                tarflags = "-jxf"
            elif f.endswith(".gz") or f.endswith(".tgz"):
                tarflags = "-zxf"
            else:
                raise RuntimeError, "unknown archive compression"
            if extractdir:
                destdir = '%s/%s' % (builddir, extractdir)
                util.execute("mkdir -p %s" % destdir)
            else:
                destdir = builddir
            util.execute("tar -C %s %s %s" % (destdir, tarflags, f))

	for file in self.sources:
            f = lookaside.findAll(self.cfg, self.laReposCache, file,
				  self.name, self.srcdirs)
	    destDir = builddir + "/" + self.theMainDir
	    util.mkdirChain(destDir)
	    shutil.copyfile(f, destDir + "/" + file)

	for (file, level, backup) in self.patches:
            # XXX handle .gz/.bz2 patch files
            f = util.findFile(file, self.srcdirs)
	    destDir = builddir + "/" + self.theMainDir
            if backup:
                backup = '-b -z %s' % backup
            util.execute('patch -d %s -p%s %s < %s' %(destDir, level, backup, f))

    def doBuild(self, buildpath):
        builddir = buildpath + "/" + self.mainDir()
	self.macros['builddir'] = builddir
        if self.build is None:
            pass
        elif type(self.build) is str:
            util.execute(self.build %self.macros)
        elif type(self.build) is tuple:
	    for bld in self.build:
                if type(bld) is str:
                    util.execute(bld %self.macros)
                else:
                    bld.doBuild(self.macros)
	else:
	    self.build.doBuild(self.macros)

    def doInstall(self, buildpath, root):
        builddir = buildpath + "/" + self.mainDir()
	self.addMacros(('builddir', builddir),
	               ('destdir', root))
        if self.install is None:
            pass
        elif type(self.install) is str:
            util.execute(self.install %self.macros)
	elif type(self.install) is tuple:
	    for inst in self.install:
                if type(inst) is str:
                    util.execute(inst %self.macros)
                else:
                    inst.doInstall(self.macros)
	else:
	    self.install.doInstall(self.macros)

    def packages(self, root):
	self.autoSpecList = []
	for spec in baseAutoSpec:
	    self.autoSpecList.append(package.PackageSpec(spec[0], spec[1]))
	# "None" will be replaced by explicit subpackage list
	self.packageSpecSet = package.PackageSpecSet(self.autoSpecList, None)
        self.packageSet = package.Auto(self.name, root, self.packageSpecSet)

    def getPackageSet(self):
        return self.packageSet

    def __init__(self, cfg, laReposCache, srcdirs, extraMacros=()):
	self.tarballs = []
	self.patches = []
	self.sources = []
	self.signatures = {}
        self.cfg = cfg
	self.laReposCache = laReposCache
	self.srcdirs = srcdirs
	self.theMainDir = self.name + "-" + self.version
	self.build = build.Make()
        self.install = build.MakeInstall()
	self.macros = Macros()
	self.addMacros = self.macros.addMacros
	self.addMacros(baseMacros)
	self.macros['name'] = self.name
	self.macros['version'] = self.version
	if extraMacros:
	    self.addMacros(extraMacros)
<commit_msg>remove unused code<commit_after>#
# Copyright (c) 2004 Specifix, Inc.
# All rights reserved
#
import imp, sys
import os
import util
import build
import package
import shutil
import types
import inspect
import lookaside
import rpmhelper
import gzip

baseMacros = (
    # Note that these macros cannot be represented as a dictionary,
    # because the items need to be added in order so that they will
    # be properly interpolated.
    #
    # paths
    ('prefix'		, '/usr'),
    ('sysconfdir'	, '/etc'),
    ('lib'              , 'lib'),  # may be overridden with 'lib64'
    ('exec_prefix'	, '%(prefix)s'),
    ('bindir'		, '%(exec_prefix)s/bin'),
    ('sbindir'		, '%(exec_prefix)s/sbin'),
    ('libdir'		, '%(exec_prefix)s/%(lib)s'),
    ('libexecdir'	, '%(exec_prefix)s/libexec'),
    ('localstatedir'	, '%(prefix)s/var'),
    ('sharedstatedir'	, '%(prefix)s/com'),
    ('includedir'	, '%(prefix)s/include'),
    ('datadir'		, '%(prefix)s/share'),
    ('mandir'		, '%(datadir)s/man'),
    ('infodir'		, '%(datadir)s/info'),
    ('docdir'		, '%(datadir)s/doc'),
    ('develdocdir'	, '%(datadir)s/develdoc'),
    # arguments/flags (empty ones are for documentation; non-existant = empty)
    ('cflags'           , '-O2'),
    ('mflags'		, ''),
    ('parallelmflags'   , ''),
    ('sysroot'		, ''),
)

crossMacros = (
    # set crossdir from cook, directly or indirectly, before adding the rest
    #('crossdir'	, 'cross-target'),
    ('prefix'		, '/opt/%(crossdir)s'),
    ('sysroot'		, '%(prefix)s/sys-root'),
    ('headerpath'	, '%(sysroot)s/usr/include')
)

# XXX TEMPORARY - remove directories such as /usr/include from this
# list when filesystem package is in place.
baseAutoSpec = (
    # automatic subpackage names and sets of regexps that define them
    # cannot be a dictionary because it is ordered; first match wins
    ('devel',
	('\.a',
	 '\.so',
	 '.*/include/.*\.h',
	 '/usr/include/',
         '/usr/include',
	 '/usr/share/man/man(2|3)/',
	 '/usr/share/man/man(2|3)',
	 '/usr/share/develdoc/',
	 '/usr/share/develdoc',
	 '/usr/share/aclocal/',
         '/usr/share/aclocal')),
    ('lib', ('.*/lib/.*\.so\..*')),
    ('doc', ('/usr/share/(doc|man|info)/',
             '/usr/share/(doc|man|info)')),
    ('locale', ('/usr/share/locale/',
                '/usr/share/locale')),
    ('runtime', ('.*',)),
)

class Macros(dict):
    def __setitem__(self, name, value):
	dict.__setitem__(self, name, value % self)

    # we want keys that don't exist to default to empty strings
    def __getitem__(self, name):
	if self.has_key(name):
	    return dict.__getitem__(self, name)
	return ''

    def addMacros(self, *macroSet):
	# must be in order; later macros in the set can depend on
	# earlier ones
	# for ease of use, we allow passing in a tuple of tuples, or
	# a simple set of tuples
	if len(macroSet) == 1 and type(macroSet[0]) is tuple:
	    # we were passed a tuple of tuples (like baseMacros)
	    macroSet = macroSet[0]
        if len(macroSet) > 0 and type(macroSet[0]) is not tuple:
            # we were passed something like ('foo', 'bar')
            macroSet = (macroSet,)
	for key, value in macroSet:
	    self[key] = value

    def copy(self):
	new = Macros()
	new.update(self)
	return new

def extractSourceFromRPM(rpm, targetfile):
    filename = os.path.basename(targetfile)
    directory = os.path.dirname(targetfile)
    r = file(rpm, 'r')
    rpmhelper.seekToData(r)
    gz = gzip.GzipFile(fileobj=r)
    (rpipe, wpipe) = os.pipe()
    pid = os.fork()
    if not pid:
	os.dup2(rpipe, 0)
	os.chdir(directory)
	os.execl('/bin/cpio', 'cpio', '-ium', filename)
	sys.exit(1)
    while 1:
	buf = gz.read(4096)
	if not buf:
	    break
	os.write(wpipe, buf)
    os.close(wpipe)
    (pid, status) = os.waitpid(pid, 0)
    if not os.WIFEXITED(status):
	raise IOError, 'cpio died extracting %s from RPM %s' \
	               %(filename, os.path.basename(rpm))
    if os.WEXITSTATUS(status):
	raise IOError, 'cpio returned failure %d extracting %s from RPM %s' \
	               %(os.WEXITSTATUS(status), filename, os.path.basename(rpm))
    if not os.path.exists(targetfile):
	raise IOError, 'failed to extract source %s from RPM %s' \
	               %(filename, os.path.basename(rpm))

class RecipeLoader(types.DictionaryType):
    def __init__(self, file):
        if file[0] != "/":
            raise IOError, "recipe file names must be absolute paths"

        self.file = os.path.basename(file).replace('.', '-')
        self.module = imp.new_module(self.file)
        sys.modules[self.file] = self.module
        f = open(file)

        exec 'from recipe import Recipe' in self.module.__dict__
        exec 'from recipe import loadRecipe' in self.module.__dict__
        exec 'import build, os, package, sys, util' in self.module.__dict__
        if sys.excepthook == util.excepthook:
            exec 'sys.excepthook = util.excepthook' in self.module.__dict__
        exec 'filename = "%s"' %(file) in self.module.__dict__
        code = compile(f.read(), file, 'exec')
        exec code in self.module.__dict__
        for (name, obj) in  self.module.__dict__.items():
            if type(obj) == types.ClassType:
                # make sure the class is derived from Recipe
                # and has a name
                if obj.__dict__.has_key('ignore'):
                    continue
                if issubclass(obj, Recipe) and obj.__dict__.has_key('name'):
                    obj.__dict__['filename'] = file
                    self[name] = obj

    def __del__(self):
        try:
            del sys.modules[self.file]
        except:
            pass

# XXX this should be extended to load a recipe from srs
def loadRecipe(file):
    callerGlobals = inspect.stack()[1][0].f_globals
    if file[0] != '/':
        recipepath = os.path.dirname(callerGlobals['filename'])
        file = recipepath + '/' + file
    recipes = RecipeLoader(file)
    for name, recipe in recipes.items():
        # XXX hack to hide parent recipes
        recipe.ignore = 1
        callerGlobals[name] = recipe
        # stash a reference to the module in the namespace
        # of the recipe that loaded it, or else it will be destroyed
        callerGlobals[os.path.basename(file).replace('.', '-')] = recipes

#def bootstrapRecipe(file, class, buildRequires):
#    loadRecipe(file) # XXX not necessary if we put boostraps in main files
#    exec """class Bootstrap%s(%s):
#	buildRequires = %s
#	name = "bootstrap-%s"
#	def setup(self):
#	    FIXMEcrossmacros(self.recipeCfg)
#	    FIXMEcrossenv
#	    FIXMEself.mainDir(class, self.version)
#	    %s.setup(self)
#    """ %(class, class, buildRequires.repr(), class, class)

class Recipe:
    buildRequires = []
    runRequires = []

    def addSignature(self, file, keyid):
	# do not search unless a gpg keyid is specified
	if not keyid:
	    return
	gpg = '%s.sig' %(file)
	c = lookaside.searchAll(self.cfg, self.laReposCache, gpg,
				self.name, self.srcdirs)
	if not c:
	    gpg = '%s.sign' %(file)
	    c = lookaside.searchAll(self.cfg, self.laReposCache,
				    gpg, self.name, self.srcdirs)
	if c:
	    if not self.signatures.has_key(file):
		self.signatures[file] = []
	    self.signatures[file].append((gpg, c, keyid))

    def addTarball(self, file, extractDir='', keyid=None):
	self.tarballs.append((file, extractDir))
	self.addSignature(file, keyid)

    def addTarballFromRPM(self, rpm, file, extractDir='', keyid=None):
	f = lookaside.searchAll(self.cfg, self.laReposCache,
			     os.path.basename(file), self.name, self.srcdirs)
	if not f:
	    r = lookaside.findAll(self.cfg, self.laReposCache, rpm,
				  self.name, self.srcdirs)
	    c = lookaside.createCacheName(self.cfg, file, self.name)
	    extractSourceFromRPM(r, c)
	    f = lookaside.findAll(self.cfg, self.laReposCache, file,
				  self.name, self.srcdirs)
	self.tarballs.append((file, extractDir))
	self.addSignature(f, keyid)

    def addPatch(self, file, level='1', backup='', keyid=None):
	self.patches.append((file, level, backup))
	self.addSignature(file, keyid)

    def addSource(self, file, keyid=None):
	self.sources.append(file)
	self.addSignature(file, keyid)

    def allSources(self):
        sources = []
        for (tarball, extractdir) in self.tarballs:
            sources.append(tarball)
        for (patch, level, backup) in self.patches:
            sources.append(patch)
	for signaturelist in self.signatures.values():
            for (gpg, cached, keyid) in signaturelist:
                sources.append(gpg)
	return sources + self.sources

    def mainDir(self, new = None):
	if new:
	    self.theMainDir = new

	return self.theMainDir

    def nameVer(self):
	return self.name + "-" + self.version

    def cleanup(self, builddir, destdir):
	shutil.rmtree(builddir)
	shutil.rmtree(destdir)

    def checkSignatures(self, filepath, file):
        if not self.signatures.has_key(file):
            return
	for (gpg, signature, keyid) in self.signatures[file]:
	    # FIXME: our own keyring
	    if os.system("gpg --no-secmem-warning --verify %s %s"
			  %(signature, filepath)):
		# FIXME: only do this if key missing, this is cheap for now
		os.system("gpg --keyserver pgp.mit.edu --recv-keys 0x %s"
		          %(keyid))
		if os.system("gpg --no-secmem-warning --verify %s %s"
			      %(signature, filepath)):
		    raise RuntimeError, "GPG signature %s failed" %(signature)

    def unpackSources(self, builddir):
	if os.path.exists(builddir):
	    shutil.rmtree(builddir)
	util.mkdirChain(builddir)
	for (file, extractdir) in self.tarballs:
            f = lookaside.findAll(self.cfg, self.laReposCache, file,
				  self.name, self.srcdirs)
	    self.checkSignatures(f, file)
            if f.endswith(".bz2"):
                tarflags = "-jxf"
            elif f.endswith(".gz") or f.endswith(".tgz"):
                tarflags = "-zxf"
            else:
                raise RuntimeError, "unknown archive compression"
            if extractdir:
                destdir = '%s/%s' % (builddir, extractdir)
                util.execute("mkdir -p %s" % destdir)
            else:
                destdir = builddir
            util.execute("tar -C %s %s %s" % (destdir, tarflags, f))

	for file in self.sources:
            f = lookaside.findAll(self.cfg, self.laReposCache, file,
				  self.name, self.srcdirs)
	    destDir = builddir + "/" + self.theMainDir
	    util.mkdirChain(destDir)
	    shutil.copyfile(f, destDir + "/" + file)

	for (file, level, backup) in self.patches:
            # XXX handle .gz/.bz2 patch files
            f = util.findFile(file, self.srcdirs)
	    destDir = builddir + "/" + self.theMainDir
            if backup:
                backup = '-b -z %s' % backup
            util.execute('patch -d %s -p%s %s < %s' %(destDir, level, backup, f))

    def doBuild(self, buildpath):
        builddir = buildpath + "/" + self.mainDir()
	self.macros['builddir'] = builddir
        if self.build is None:
            pass
        elif type(self.build) is str:
            util.execute(self.build %self.macros)
        elif type(self.build) is tuple:
	    for bld in self.build:
                if type(bld) is str:
                    util.execute(bld %self.macros)
                else:
                    bld.doBuild(self.macros)
	else:
	    self.build.doBuild(self.macros)

    def doInstall(self, buildpath, root):
        builddir = buildpath + "/" + self.mainDir()
	self.addMacros(('builddir', builddir),
	               ('destdir', root))
        if self.install is None:
            pass
        elif type(self.install) is str:
            util.execute(self.install %self.macros)
	elif type(self.install) is tuple:
	    for inst in self.install:
                if type(inst) is str:
                    util.execute(inst %self.macros)
                else:
                    inst.doInstall(self.macros)
	else:
	    self.install.doInstall(self.macros)

    def packages(self, root):
	self.autoSpecList = []
	for spec in baseAutoSpec:
	    self.autoSpecList.append(package.PackageSpec(spec[0], spec[1]))
	# "None" will be replaced by explicit subpackage list
	self.packageSpecSet = package.PackageSpecSet(self.autoSpecList, None)
        self.packageSet = package.Auto(self.name, root, self.packageSpecSet)

    def getPackageSet(self):
        return self.packageSet

    def __init__(self, cfg, laReposCache, srcdirs, extraMacros=()):
	self.tarballs = []
	self.patches = []
	self.sources = []
	self.signatures = {}
        self.cfg = cfg
	self.laReposCache = laReposCache
	self.srcdirs = srcdirs
	self.theMainDir = self.name + "-" + self.version
	self.build = build.Make()
        self.install = build.MakeInstall()
	self.macros = Macros()
	self.addMacros = self.macros.addMacros
	self.addMacros(baseMacros)
	self.macros['name'] = self.name
	self.macros['version'] = self.version
	if extraMacros:
	    self.addMacros(extraMacros)
<|endoftext|>


<commit_before>import unittest
from bubbles.errors import *
from bubbles.core import *
from bubbles import DataObject
import bubbles.iterator

# FIXME: clean this up
import inspect

def default(ctx, left):
    pass

def unary(ctx, left):
    pass

def binary(ctx, left, right):
    pass

class DummyDataObject(DataObject):
    def __init__(self, reps=None, data=None):
        self.reps = reps or []
        self.data = data

    def representations(self):
        return self.reps

class TextObject(DataObject):
    def __init__(self, string):
        self.string = string

    def representations(self):
        return ["rows", "text"]

    def rows(self):
        return iter(self.string)

    def text(self):
        return self.string

class KernelTestCase(unittest.TestCase):
    def test_match(self):
        self.assertTrue(Signature("sql").matches("sql"))
        self.assertTrue(Signature("*").matches("sql"))
        self.assertTrue(Signature("sql[]").matches("sql[]"))
        self.assertTrue(Signature("*[]").matches("sql[]"))

        self.assertFalse(Signature("sql").matches("rows"))
        self.assertFalse(Signature("sql").matches("sql[]"))

    def test_common_reps(self):
        objs = [
                DummyDataObject(["a", "b", "c"]),
                DummyDataObject(["a", "b", "d"]),
                DummyDataObject(["b", "d", "e"])
            ]
        self.assertEqual(["b"], list(common_representations(*objs)))

        objs = [
                DummyDataObject(["a", "b", "c"]),
                DummyDataObject(["a", "b", "d"]),
                DummyDataObject(["d", "d", "e"])
            ]
        self.assertEqual([], list(common_representations(*objs)))

    def test_prototype(self):
        proto = Signature("sql", "sql")
        match = Signature("*", "*")
        self.assertEqual(match, proto.as_prototype())

        proto = Signature("sql[]", "sql")
        match = Signature("*[]", "*")
        self.assertEqual(match, proto.as_prototype())

        proto = Signature("*[]", "*")
        match = Signature("*[]", "*")
        self.assertEqual(match, proto.as_prototype())

    def test_create_op(self):
        def fun(ctx, a, b):
            pass

        self.assertFalse(is_operation(fun))

        op = Operation(fun, Signature("sql", "rows"))

        self.assertEqual("fun", op.name)
        self.assertEqual("sql", op.signature[0])

        op2 = operation("sql", "rows")(fun)

        self.assertEqual("fun", op.name)
        self.assertEqual("sql", op.signature[0])

        self.assertEqual(op.function, op2.function)
        self.assertEqual(op.name, op2.name)
        self.assertEqual(op.signature, op2.signature)
        self.assertEqual(op, op2)

    def test_register(self):
        def func(ctx, obj):
            pass

        c = OperationContext()
        self.assertFalse(c.operations["func"])

        c.add_operation(Operation(func, Signature("sql")))
        self.assertTrue(c.operations["func"])

        c.add_operation(Operation(func, Signature("sql"), name="other"))
        self.assertTrue(c.operations["other"])

        with self.assertRaises(ArgumentError):
            c.add_operation(Operation(func, Signature("sql")))

    def test_register_decorated(self):
        def func(ctx, obj):
            pass

        c = OperationContext()

        op = Operation(func, ["rows"])
        c.add_operation(op)
        self.assertTrue(c.operations["func"])

        with self.assertRaises(ArgumentError):
            c.add_operation(op)

    def test_prototype(self):
        def join(ctx, master, detail, master_key, detail_key):
            pass

        c = OperationContext()

        op = Operation(join, Signature("rows", "rows"))
        c.add_operation(op)

        proto = c.operation_prototype("join")
        self.assertEqual(2, proto.operand_count)
        self.assertSequenceEqual(["master", "detail"], proto.operands)
        self.assertSequenceEqual(["master_key", "detail_key"],
                                 proto.parameters)

    def test_extract_signatures(self):
        obj = DummyDataObject(["rows", "sql"])
        self.assertEqual( [["rows", "sql"]], extract_signatures(obj))

        obj = DummyDataObject(["rows", "sql"])
        extr = extract_signatures([obj])
        self.assertEqual( [["rows[]", "sql[]"]], extr)

    def test_lookup(self):
        c = OperationContext()

        obj_sql = DummyDataObject(["sql"])
        obj_rows = DummyDataObject(["rows"])

        c.add_operation(Operation(unary, signature=Signature("sql")))
        c.add_operation(Operation(default, name="unary",
                                signature=Signature("*")))

        match = c.lookup_operation("unary", obj_sql)
        self.assertEqual(unary, match.function)

        match = c.lookup_operation("unary", obj_rows)
        self.assertEqual(default, match.function)

        with self.assertRaises(OperationError):
            c.lookup_operation("foo", obj_sql)

        with self.assertRaises(OperationError):
            c.lookup_operation("unary", obj_sql, obj_sql)

    def test_lookup_additional_args(self):
        def func(ctx, obj, value):
            pass

        c = OperationContext()
        c.add_operation(Operation(func,
                                signature=Signature("rows")))

        obj = DummyDataObject(["rows"])

        c.o.func(obj, 1)

    def test_comparison(self):
        sig1 = Signature("a", "b", "c")
        sig2 = Signature("a", "b", "c")
        sig3 = Signature("a", "b")

        self.assertTrue(sig1 == sig1)
        self.assertTrue(sig1 == sig2)
        self.assertFalse(sig1 == sig3)

        self.assertTrue(sig1 == ["a", "b", "c"])
        self.assertFalse(sig1 == ["a", "b"])

    def test_delete(self):
        c = OperationContext()
        obj = DummyDataObject(["rows"])

        c.add_operation(Operation(unary, signature=Signature("rows")))
        c.add_operation(Operation(default, name="unary", signature=Signature("*")))

        match = c.lookup_operation("unary", obj)
        self.assertEqual(unary, match.function)

        c.remove_operation("unary", ["rows"])
        match = c.lookup_operation("unary", obj)
        self.assertEqual(default, match.function)

        c.remove_operation("unary")
        with self.assertRaises(OperationError):
            c.lookup_operation("unary", obj)

    def test_running(self):
        def func_text(ctx, obj):
            text = obj.text()
            return list(text.upper())

        def func_rows(ctx, obj):
            rows = obj.rows()
            text = "".join(rows)
            return list(text.upper())

        c = OperationContext()
        c.add_operation(Operation(func_text, name="upper",
                                signature=Signature("text")))
        c.add_operation(Operation(func_rows, name="upper",
                                signature=Signature("rows")))

        obj = TextObject("windchimes")

        result = c.o.upper(obj)
        self.assertEqual(list("WINDCHIMES"), result)
        # func = om.match("upper")

    def test_retry(self):
        @operation("sql", "sql", name="join")
        def join_sql(ctx, l, r):
            if l.data == r.data:
                return "SQL"
            else:
                raise RetryOperation(["sql", "rows"])

        @operation("sql", "rows", name="join")
        def join_iter(ctx, l, r):
            return "ITERATOR"

        @operation("sql", "sql")
        def endless(ctx, l, r):
            raise RetryOperation(["sql", "sql"])

        local = DummyDataObject(["sql", "rows"], "local")
        remote = DummyDataObject(["sql", "rows"], "remote")

        c = OperationContext()
        c.add_operation(join_sql)
        c.add_operation(join_iter)

        result = c.o.join(local, local)
        self.assertEqual(result, "SQL")

        result = c.o.join(local, remote)
        self.assertEqual(result, "ITERATOR")

        c.add_operation(endless)
        with self.assertRaises(RetryError):
            result = c.o.endless(local, local)

    def test_priority(self):
        objsql = DummyDataObject(["sql", "rows"])
        objrows = DummyDataObject(["rows", "sql"])

        def fsql(ctx, obj):
            pass
        def frows(ctx, obj):
            pass

        c = OperationContext()
        c.add_operation(Operation(fsql, name="meditate",
                                    signature=Signature("sql")))
        c.add_operation(Operation(frows, name="meditate",
                                    signature=Signature("rows")))

        self.assertEqual(fsql, c.lookup_operation("meditate", objsql).function)
        self.assertEqual(frows, c.lookup_operation("meditate",
                                                            objrows).function)

        # Reverse order of registration, expect the same result
        c = OperationContext()
        c.add_operation(Operation(frows, name="meditate",
                                signature=Signature("rows")))
        c.add_operation(Operation(fsql, name="meditate",
                                signature=Signature("sql")))

        self.assertEqual(fsql, c.lookup_operation("meditate", objsql).function)
        self.assertEqual(frows, c.lookup_operation("meditate",
                                                            objrows).function)

if __name__ == "__main__":
    unittest.main()
<commit_msg>added test that confirms that nested retry works, closes #4<commit_after>import unittest
from bubbles.errors import *
from bubbles.core import *
from bubbles import DataObject
import bubbles.iterator

# FIXME: clean this up
import inspect

def default(ctx, left):
    pass

def unary(ctx, left):
    pass

def binary(ctx, left, right):
    pass

class DummyDataObject(DataObject):
    def __init__(self, reps=None, data=None):
        self.reps = reps or []
        self.data = data

    def representations(self):
        return self.reps

class TextObject(DataObject):
    def __init__(self, string):
        self.string = string

    def representations(self):
        return ["rows", "text"]

    def rows(self):
        return iter(self.string)

    def text(self):
        return self.string

class KernelTestCase(unittest.TestCase):
    def test_match(self):
        self.assertTrue(Signature("sql").matches("sql"))
        self.assertTrue(Signature("*").matches("sql"))
        self.assertTrue(Signature("sql[]").matches("sql[]"))
        self.assertTrue(Signature("*[]").matches("sql[]"))

        self.assertFalse(Signature("sql").matches("rows"))
        self.assertFalse(Signature("sql").matches("sql[]"))

    def test_common_reps(self):
        objs = [
                DummyDataObject(["a", "b", "c"]),
                DummyDataObject(["a", "b", "d"]),
                DummyDataObject(["b", "d", "e"])
            ]
        self.assertEqual(["b"], list(common_representations(*objs)))

        objs = [
                DummyDataObject(["a", "b", "c"]),
                DummyDataObject(["a", "b", "d"]),
                DummyDataObject(["d", "d", "e"])
            ]
        self.assertEqual([], list(common_representations(*objs)))

    def test_prototype(self):
        proto = Signature("sql", "sql")
        match = Signature("*", "*")
        self.assertEqual(match, proto.as_prototype())

        proto = Signature("sql[]", "sql")
        match = Signature("*[]", "*")
        self.assertEqual(match, proto.as_prototype())

        proto = Signature("*[]", "*")
        match = Signature("*[]", "*")
        self.assertEqual(match, proto.as_prototype())

    def test_create_op(self):
        def fun(ctx, a, b):
            pass

        self.assertFalse(is_operation(fun))

        op = Operation(fun, Signature("sql", "rows"))

        self.assertEqual("fun", op.name)
        self.assertEqual("sql", op.signature[0])

        op2 = operation("sql", "rows")(fun)

        self.assertEqual("fun", op.name)
        self.assertEqual("sql", op.signature[0])

        self.assertEqual(op.function, op2.function)
        self.assertEqual(op.name, op2.name)
        self.assertEqual(op.signature, op2.signature)
        self.assertEqual(op, op2)

    def test_register(self):
        def func(ctx, obj):
            pass

        c = OperationContext()
        self.assertFalse(c.operations["func"])

        c.add_operation(Operation(func, Signature("sql")))
        self.assertTrue(c.operations["func"])

        c.add_operation(Operation(func, Signature("sql"), name="other"))
        self.assertTrue(c.operations["other"])

        with self.assertRaises(ArgumentError):
            c.add_operation(Operation(func, Signature("sql")))

    def test_register_decorated(self):
        def func(ctx, obj):
            pass

        c = OperationContext()

        op = Operation(func, ["rows"])
        c.add_operation(op)
        self.assertTrue(c.operations["func"])

        with self.assertRaises(ArgumentError):
            c.add_operation(op)

    def test_prototype(self):
        def join(ctx, master, detail, master_key, detail_key):
            pass

        c = OperationContext()

        op = Operation(join, Signature("rows", "rows"))
        c.add_operation(op)

        proto = c.operation_prototype("join")
        self.assertEqual(2, proto.operand_count)
        self.assertSequenceEqual(["master", "detail"], proto.operands)
        self.assertSequenceEqual(["master_key", "detail_key"],
                                 proto.parameters)

    def test_extract_signatures(self):
        obj = DummyDataObject(["rows", "sql"])
        self.assertEqual( [["rows", "sql"]], extract_signatures(obj))

        obj = DummyDataObject(["rows", "sql"])
        extr = extract_signatures([obj])
        self.assertEqual( [["rows[]", "sql[]"]], extr)

    def test_lookup(self):
        c = OperationContext()

        obj_sql = DummyDataObject(["sql"])
        obj_rows = DummyDataObject(["rows"])

        c.add_operation(Operation(unary, signature=Signature("sql")))
        c.add_operation(Operation(default, name="unary",
                                signature=Signature("*")))

        match = c.lookup_operation("unary", obj_sql)
        self.assertEqual(unary, match.function)

        match = c.lookup_operation("unary", obj_rows)
        self.assertEqual(default, match.function)

        with self.assertRaises(OperationError):
            c.lookup_operation("foo", obj_sql)

        with self.assertRaises(OperationError):
            c.lookup_operation("unary", obj_sql, obj_sql)

    def test_lookup_additional_args(self):
        def func(ctx, obj, value):
            pass

        c = OperationContext()
        c.add_operation(Operation(func,
                                signature=Signature("rows")))

        obj = DummyDataObject(["rows"])

        c.o.func(obj, 1)

    def test_comparison(self):
        sig1 = Signature("a", "b", "c")
        sig2 = Signature("a", "b", "c")
        sig3 = Signature("a", "b")

        self.assertTrue(sig1 == sig1)
        self.assertTrue(sig1 == sig2)
        self.assertFalse(sig1 == sig3)

        self.assertTrue(sig1 == ["a", "b", "c"])
        self.assertFalse(sig1 == ["a", "b"])

    def test_delete(self):
        c = OperationContext()
        obj = DummyDataObject(["rows"])

        c.add_operation(Operation(unary, signature=Signature("rows")))
        c.add_operation(Operation(default, name="unary", signature=Signature("*")))

        match = c.lookup_operation("unary", obj)
        self.assertEqual(unary, match.function)

        c.remove_operation("unary", ["rows"])
        match = c.lookup_operation("unary", obj)
        self.assertEqual(default, match.function)

        c.remove_operation("unary")
        with self.assertRaises(OperationError):
            c.lookup_operation("unary", obj)

    def test_running(self):
        def func_text(ctx, obj):
            text = obj.text()
            return list(text.upper())

        def func_rows(ctx, obj):
            rows = obj.rows()
            text = "".join(rows)
            return list(text.upper())

        c = OperationContext()
        c.add_operation(Operation(func_text, name="upper",
                                signature=Signature("text")))
        c.add_operation(Operation(func_rows, name="upper",
                                signature=Signature("rows")))

        obj = TextObject("windchimes")

        result = c.o.upper(obj)
        self.assertEqual(list("WINDCHIMES"), result)
        # func = om.match("upper")

    def test_retry(self):
        @operation("sql", "sql", name="join")
        def join_sql(ctx, l, r):
            if l.data == r.data:
                return "SQL"
            else:
                raise RetryOperation(["sql", "rows"])

        @operation("sql", "rows", name="join")
        def join_iter(ctx, l, r):
            return "ITERATOR"

        @operation("sql", "sql")
        def endless(ctx, l, r):
            raise RetryOperation(["sql", "sql"])

        local = DummyDataObject(["sql", "rows"], "local")
        remote = DummyDataObject(["sql", "rows"], "remote")

        c = OperationContext()
        c.add_operation(join_sql)
        c.add_operation(join_iter)

        result = c.o.join(local, local)
        self.assertEqual(result, "SQL")

        result = c.o.join(local, remote)
        self.assertEqual(result, "ITERATOR")

        c.add_operation(endless)
        with self.assertRaises(RetryError):
            result = c.o.endless(local, local)

    def test_retry_nested(self):
        """Test whether failed nested operation fails correctly (Because of
        Issue #4)."""

        @operation("sql")
        def aggregate(ctx, obj, fail):
            if fail:
                raise RetryOperation(["rows"])
            else:
                obj.data += "-SQL-"
            return obj

        @operation("rows", name="aggregate")
        def aggregate_rows(ctx, obj, fail):
            obj.data += "-ROWS-"
            return obj

        @operation("sql")
        def window_aggregate(ctx, obj, fail):
            obj.data += "START"
            ctx.o.aggregate(obj, fail)
            obj.data += "END"

        c = OperationContext()
        c.add_operation(aggregate)
        c.add_operation(aggregate_rows)
        c.add_operation(window_aggregate)

        # Expected order:
        # 1. window_aggregate is called
        # 2. sql aggregate is called, but fails
        # 3. row aggregate is called
        # 4. window aggregate continues

        obj = DummyDataObject(["sql"], "")

        c.o.window_aggregate(obj, fail=True)
        self.assertEqual("START-ROWS-END", obj.data)

        obj.data = ""
        c.o.window_aggregate(obj, fail=False)
        self.assertEqual("START-SQL-END", obj.data)

    def test_priority(self):
        objsql = DummyDataObject(["sql", "rows"])
        objrows = DummyDataObject(["rows", "sql"])

        def fsql(ctx, obj):
            pass
        def frows(ctx, obj):
            pass

        c = OperationContext()
        c.add_operation(Operation(fsql, name="meditate",
                                    signature=Signature("sql")))
        c.add_operation(Operation(frows, name="meditate",
                                    signature=Signature("rows")))

        self.assertEqual(fsql, c.lookup_operation("meditate", objsql).function)
        self.assertEqual(frows, c.lookup_operation("meditate",
                                                            objrows).function)

        # Reverse order of registration, expect the same result
        c = OperationContext()
        c.add_operation(Operation(frows, name="meditate",
                                signature=Signature("rows")))
        c.add_operation(Operation(fsql, name="meditate",
                                signature=Signature("sql")))

        self.assertEqual(fsql, c.lookup_operation("meditate", objsql).function)
        self.assertEqual(frows, c.lookup_operation("meditate",
                                                            objrows).function)

if __name__ == "__main__":
    unittest.main()
<|endoftext|>


<commit_before># Imports
from django.http import HttpResponse
from django.template import loader
from django.shortcuts import redirect

##
# Handle 404 Errors
# @param request WSGIRequest list with all HTTP Request
def error404(request):

    # Generate Content for this view
    template = loader.get_template('404.html')

    context = {
        'referrer': request.META.get('HTTP_REFERER', '/'),
    }

    # 3. Return Template for this view + Data
    return HttpResponse(content=template.render(context),
                        content_type='text/html; charset=utf-8', status=404)

def error500(request):

    # Generate Content for this view
    template = loader.get_template('500.html')

    context = {
        'referrer': request.META.get('HTTP_REFERER', '/'),
    }

    # Return Template for this view + Data
    return HttpResponse(content=template.render(context),
                        content_type='text/html; charset=utf-8', status=500)

def home(request):
    return redirect('/api')
<commit_msg>Updated isort<commit_after># Imports
from django.http import HttpResponse
from django.shortcuts import redirect
from django.template import loader

##
# Handle 404 Errors
# @param request WSGIRequest list with all HTTP Request
def error404(request):

    # Generate Content for this view
    template = loader.get_template('404.html')

    context = {
        'referrer': request.META.get('HTTP_REFERER', '/'),
    }

    # 3. Return Template for this view + Data
    return HttpResponse(content=template.render(context),
                        content_type='text/html; charset=utf-8', status=404)

def error500(request):

    # Generate Content for this view
    template = loader.get_template('500.html')

    context = {
        'referrer': request.META.get('HTTP_REFERER', '/'),
    }

    # Return Template for this view + Data
    return HttpResponse(content=template.render(context),
                        content_type='text/html; charset=utf-8', status=500)

def home(request):
    return redirect('/api')
<|endoftext|>


<commit_before>"""This directory is setup with configurations to run the main functional test.

It exercises a full analysis pipeline on a smaller subset of data.
"""
import os
import subprocess
import unittest
import shutil
import contextlib
import collections
import functools

from nose import SkipTest
from nose.plugins.attrib import attr
import yaml

from bcbio.pipeline.config_utils import load_system_config

@contextlib.contextmanager
def make_workdir():
    remove_old_dir = True
    #remove_old_dir = False
    dirname = os.path.join(os.path.dirname(__file__), "test_automated_output")
    if remove_old_dir:
        if os.path.exists(dirname):
            shutil.rmtree(dirname)
        os.makedirs(dirname)
    orig_dir = os.getcwd()
    try:
        os.chdir(dirname)
        yield dirname
    finally:
        os.chdir(orig_dir)

def expected_failure(test):
    """Small decorator to mark tests as expected failure.
    Useful for tests that are work-in-progress.
    """
    @functools.wraps(test)
    def inner(*args, **kwargs):
        try:
            test(*args, **kwargs)
        except Exception:
            raise SkipTest
        else:
            raise AssertionError('Failure expected')
    return inner

def get_post_process_yaml(data_dir, workdir):
    try:
        from bcbiovm.docker.defaults import get_datadir
        datadir = get_datadir()
        system = os.path.join(datadir, "galaxy", "bcbio_system.yaml") if datadir else None
    except ImportError:
        system = None
    if system is None or not os.path.exists(system):
        try:
            _, system = load_system_config("bcbio_system.yaml")
        except ValueError:
            system = None
    sample = os.path.join(data_dir, "post_process-sample.yaml")
    std = os.path.join(data_dir, "post_process.yaml")
    if os.path.exists(std):
        return std
    elif system and os.path.exists(system):
        # create local config pointing to reduced genomes
        test_system = os.path.join(workdir, os.path.basename(system))
        with open(system) as in_handle:
            config = yaml.load(in_handle)
            config["galaxy_config"] = os.path.join(data_dir, "universe_wsgi.ini")
            with open(test_system, "w") as out_handle:
                yaml.dump(config, out_handle)
        return test_system
    else:
        return sample

class AutomatedAnalysisTest(unittest.TestCase):
    """Setup a full automated analysis and run the pipeline.
    """
    def setUp(self):
        self.data_dir = os.path.join(os.path.dirname(__file__), "data", "automated")

    def _install_test_files(self, data_dir):
        """Download required sequence and reference files.
        """
        DlInfo = collections.namedtuple("DlInfo", "fname dirname version")
        download_data = [DlInfo("110106_FC70BUKAAXX.tar.gz", None, None),
                         DlInfo("genomes_automated_test.tar.gz", "genomes", 13),
                         DlInfo("110907_ERP000591.tar.gz", None, None),
                         DlInfo("100326_FC6107FAAXX.tar.gz", None, 5),
                         DlInfo("tcga_benchmark.tar.gz", None, 2)]
        for dl in download_data:
            url = "http://chapmanb.s3.amazonaws.com/{fname}".format(fname=dl.fname)
            dirname = os.path.join(data_dir, os.pardir,
                                   dl.fname.replace(".tar.gz", "") if dl.dirname is None
                                   else dl.dirname)
            if os.path.exists(dirname) and dl.version is not None:
                version_file = os.path.join(dirname, "VERSION")
                is_old = True
                if os.path.exists(version_file):
                    with open(version_file) as in_handle:
                        version = int(in_handle.read())
                    is_old = version < dl.version
                if is_old:
                    shutil.rmtree(dirname)
            if not os.path.exists(dirname):
                self._download_to_dir(url, dirname)

    def _download_to_dir(self, url, dirname):
        print dirname
        cl = ["wget", url]
        subprocess.check_call(cl)
        cl = ["tar", "-xzvpf", os.path.basename(url)]
        subprocess.check_call(cl)
        os.rename(os.path.basename(dirname), dirname)
        os.remove(os.path.basename(url))

    @attr(speed=3)
    def IGNOREtest_3_full_pipeline(self):
        """Run full automated analysis pipeline with multiplexing.

        XXX Multiplexing not supporting in latest versions.
        """
        self._install_test_files(self.data_dir)
        with make_workdir() as workdir:
            cl = ["bcbio_nextgen.py",
                  get_post_process_yaml(self.data_dir, workdir),
                  os.path.join(self.data_dir, os.pardir, "110106_FC70BUKAAXX"),
                  os.path.join(self.data_dir, "run_info.yaml")]
            subprocess.check_call(cl)

    @attr(speed=3)
    def IGNOREtest_4_empty_fastq(self):
        """Handle analysis of empty fastq inputs from failed runs.

        XXX Multiplexing not supporting in latest versions.
        """
        with make_workdir() as workdir:
            cl = ["bcbio_nextgen.py",
                  get_post_process_yaml(self.data_dir, workdir),
                  os.path.join(self.data_dir, os.pardir, "110221_empty_FC12345AAXX"),
                  os.path.join(self.data_dir, "run_info-empty.yaml")]
            subprocess.check_call(cl)

    @attr(stranded=True)
    def test_2_stranded(self):
        """Run an RNA-seq analysis with TopHat and generate gene-level counts.
        """
        self._install_test_files(self.data_dir)
        with make_workdir() as workdir:
            cl = ["bcbio_nextgen.py",
                  get_post_process_yaml(self.data_dir, workdir),
                  os.path.join(self.data_dir, os.pardir, "test_stranded"),
                  os.path.join(self.data_dir, "run_info-stranded.yaml")]
            subprocess.check_call(cl)

    @attr(rnaseq=True)
    @attr(tophat=True)
    def test_2_rnaseq(self):
        """Run an RNA-seq analysis with TopHat and generate gene-level counts.
        """
        self._install_test_files(self.data_dir)
        with make_workdir() as workdir:
            cl = ["bcbio_nextgen.py",
                  get_post_process_yaml(self.data_dir, workdir),
                  os.path.join(self.data_dir, os.pardir, "110907_ERP000591"),
                  os.path.join(self.data_dir, "run_info-rnaseq.yaml")]
            subprocess.check_call(cl)

    @expected_failure
    @attr(fusion=True)
    def test_2_fusion(self):
        """Run an RNA-seq analysis and test fusion genes
        """
        self._install_test_files(self.data_dir)
        with make_workdir() as workdir:
            cl = ["bcbio_nextgen.py",
                  "/usr/local/share/bcbio-nextgen/galaxy/bcbio_system.yaml",
                  # get_post_process_yaml(self.data_dir, workdir),
                  os.path.join(self.data_dir, os.pardir, "test_fusion"),
                  os.path.join(self.data_dir, "run_info-fusion.yaml")]
            subprocess.check_call(cl)

    @attr(rnaseq=True)
    @attr(star=True)
    def test_2_star(self):
        """Run an RNA-seq analysis with STAR and generate gene-level counts.
        """
        self._install_test_files(self.data_dir)
        with make_workdir() as workdir:
            cl = ["bcbio_nextgen.py",
                  get_post_process_yaml(self.data_dir, workdir),
                  os.path.join(self.data_dir, os.pardir, "110907_ERP000591"),
                  os.path.join(self.data_dir, "run_info-star.yaml")]
            subprocess.check_call(cl)

    @attr(explant=True)
    def test_explant(self):
        """
        Run an explant RNA-seq analysis with TopHat and generate gene-level counts.
        """
        self._install_test_files(self.data_dir)
        with make_workdir() as workdir:
            cl = ["bcbio_nextgen.py",
                  get_post_process_yaml(self.data_dir, workdir),
                  os.path.join(self.data_dir, os.pardir, "1_explant"),
                  os.path.join(self.data_dir, "run_info-explant.yaml")]
            subprocess.check_call(cl)

    @attr(chipseq=True)
    def test_chipseq(self):
        """
        Run a chip-seq alignment with Bowtie2
        """
        self._install_test_files(self.data_dir)
        with make_workdir() as workdir:
            cl = ["bcbio_nextgen.py",
                  get_post_process_yaml(self.data_dir, workdir),
                  os.path.join(self.data_dir, os.pardir, "test_chipseq"),
                  os.path.join(self.data_dir, "run_info-chipseq.yaml")]
            subprocess.check_call(cl)

    @attr(speed=1)
    @attr(ensemble=True)
    def test_1_variantcall(self):
        """Test variant calling with GATK pipeline.
        """
        self._install_test_files(self.data_dir)
        with make_workdir() as workdir:
            cl = ["bcbio_nextgen.py",
                  get_post_process_yaml(self.data_dir, workdir),
                  os.path.join(self.data_dir, os.pardir, "100326_FC6107FAAXX"),
                  os.path.join(self.data_dir, "run_info-variantcall.yaml")]
            subprocess.check_call(cl)

    @attr(speed=1)
    @attr(devel=True)
    def test_5_bam(self):
        """Allow BAM files as input to pipeline.
        """
        self._install_test_files(self.data_dir)
        with make_workdir() as workdir:
            cl = ["bcbio_nextgen.py",
                  get_post_process_yaml(self.data_dir, workdir),
                  os.path.join(self.data_dir, os.pardir, "100326_FC6107FAAXX"),
                  os.path.join(self.data_dir, "run_info-bam.yaml")]
            subprocess.check_call(cl)

    @attr(speed=2)
    def test_6_bamclean(self):
        """Clean problem BAM input files that do not require alignment.
        """
        self._install_test_files(self.data_dir)
        with make_workdir() as workdir:
            cl = ["bcbio_nextgen.py",
                  get_post_process_yaml(self.data_dir, workdir),
                  os.path.join(self.data_dir, os.pardir, "100326_FC6107FAAXX"),
                  os.path.join(self.data_dir, "run_info-bamclean.yaml")]
            subprocess.check_call(cl)

    @attr(speed=2)
    @attr(cancer=True)
    def test_7_cancer(self):
        """Test paired tumor-normal calling using multiple calling approaches: MuTect, VarScan, FreeBayes.
        """
        self._install_test_files(self.data_dir)
        with make_workdir() as workdir:
            cl = ["bcbio_nextgen.py",
                  get_post_process_yaml(self.data_dir, workdir),
                  os.path.join(self.data_dir, os.pardir, "tcga_benchmark"),
                  os.path.join(self.data_dir, "run_info-cancer.yaml")]
            subprocess.check_call(cl)

    @attr(cancer=True)
    def test_7_cancer_nonormal(self):
        """Test cancer calling without normal samples or with normal VCF panels.
        """
        self._install_test_files(self.data_dir)
        with make_workdir() as workdir:
            cl = ["bcbio_nextgen.py",
                  get_post_process_yaml(self.data_dir, workdir),
                  os.path.join(self.data_dir, "run_info-cancer2.yaml")]
            subprocess.check_call(cl)

    @attr(speed=1)
    @attr(template=True)
    def test_8_template(self):
        """Create a project template from input files and metadata configuration.
        """
        self._install_test_files(self.data_dir)
        fc_dir = os.path.join(self.data_dir, os.pardir, "100326_FC6107FAAXX")
        with make_workdir() as workdir:
            cl = ["bcbio_nextgen.py", "-w", "template", "freebayes-variant",
                  os.path.join(fc_dir, "100326.csv"),
                  os.path.join(fc_dir, "7_100326_FC6107FAAXX_1_fastq.txt"),
                  os.path.join(fc_dir, "7_100326_FC6107FAAXX_2_fastq.txt"),
                  os.path.join(fc_dir, "8_100326_FC6107FAAXX.bam")]
            subprocess.check_call(cl)

    @attr(docker=True)
    def test_docker(self):
        """Run an analysis with code and tools inside a docker container.

        Requires https://github.com/chapmanb/bcbio-nextgen-vm
        """
        self._install_test_files(self.data_dir)
        with make_workdir() as workdir:
            cl = ["bcbio_vm.py", "run",
                  "--systemconfig=%s" % get_post_process_yaml(self.data_dir, workdir),
                  "--fcdir=%s" % os.path.join(self.data_dir, os.pardir, "100326_FC6107FAAXX"),
                  os.path.join(self.data_dir, "run_info-bam.yaml")]
            subprocess.check_call(cl)

    @attr(docker_ipython=True)
    def test_docker_ipython(self):
        """Run an analysis with code and tools inside a docker container, driven via IPython.

        Requires https://github.com/chapmanb/bcbio-nextgen-vm
        """
        self._install_test_files(self.data_dir)
        with make_workdir() as workdir:
            cl = ["bcbio_vm.py", "ipython",
                  "--systemconfig=%s" % get_post_process_yaml(self.data_dir, workdir),
                  "--fcdir=%s" % os.path.join(self.data_dir, os.pardir, "100326_FC6107FAAXX"),
                  os.path.join(self.data_dir, "run_info-bam.yaml"),
                  "lsf", "localrun"]
            subprocess.check_call(cl)
<commit_msg>Add target for only running multiple caller test. #340<commit_after>"""This directory is setup with configurations to run the main functional test.

It exercises a full analysis pipeline on a smaller subset of data.
"""
import os
import subprocess
import unittest
import shutil
import contextlib
import collections
import functools

from nose import SkipTest
from nose.plugins.attrib import attr
import yaml

from bcbio.pipeline.config_utils import load_system_config

@contextlib.contextmanager
def make_workdir():
    remove_old_dir = True
    #remove_old_dir = False
    dirname = os.path.join(os.path.dirname(__file__), "test_automated_output")
    if remove_old_dir:
        if os.path.exists(dirname):
            shutil.rmtree(dirname)
        os.makedirs(dirname)
    orig_dir = os.getcwd()
    try:
        os.chdir(dirname)
        yield dirname
    finally:
        os.chdir(orig_dir)

def expected_failure(test):
    """Small decorator to mark tests as expected failure.
    Useful for tests that are work-in-progress.
    """
    @functools.wraps(test)
    def inner(*args, **kwargs):
        try:
            test(*args, **kwargs)
        except Exception:
            raise SkipTest
        else:
            raise AssertionError('Failure expected')
    return inner

def get_post_process_yaml(data_dir, workdir):
    try:
        from bcbiovm.docker.defaults import get_datadir
        datadir = get_datadir()
        system = os.path.join(datadir, "galaxy", "bcbio_system.yaml") if datadir else None
    except ImportError:
        system = None
    if system is None or not os.path.exists(system):
        try:
            _, system = load_system_config("bcbio_system.yaml")
        except ValueError:
            system = None
    sample = os.path.join(data_dir, "post_process-sample.yaml")
    std = os.path.join(data_dir, "post_process.yaml")
    if os.path.exists(std):
        return std
    elif system and os.path.exists(system):
        # create local config pointing to reduced genomes
        test_system = os.path.join(workdir, os.path.basename(system))
        with open(system) as in_handle:
            config = yaml.load(in_handle)
            config["galaxy_config"] = os.path.join(data_dir, "universe_wsgi.ini")
            with open(test_system, "w") as out_handle:
                yaml.dump(config, out_handle)
        return test_system
    else:
        return sample

class AutomatedAnalysisTest(unittest.TestCase):
    """Setup a full automated analysis and run the pipeline.
    """
    def setUp(self):
        self.data_dir = os.path.join(os.path.dirname(__file__), "data", "automated")

    def _install_test_files(self, data_dir):
        """Download required sequence and reference files.
        """
        DlInfo = collections.namedtuple("DlInfo", "fname dirname version")
        download_data = [DlInfo("110106_FC70BUKAAXX.tar.gz", None, None),
                         DlInfo("genomes_automated_test.tar.gz", "genomes", 13),
                         DlInfo("110907_ERP000591.tar.gz", None, None),
                         DlInfo("100326_FC6107FAAXX.tar.gz", None, 5),
                         DlInfo("tcga_benchmark.tar.gz", None, 2)]
        for dl in download_data:
            url = "http://chapmanb.s3.amazonaws.com/{fname}".format(fname=dl.fname)
            dirname = os.path.join(data_dir, os.pardir,
                                   dl.fname.replace(".tar.gz", "") if dl.dirname is None
                                   else dl.dirname)
            if os.path.exists(dirname) and dl.version is not None:
                version_file = os.path.join(dirname, "VERSION")
                is_old = True
                if os.path.exists(version_file):
                    with open(version_file) as in_handle:
                        version = int(in_handle.read())
                    is_old = version < dl.version
                if is_old:
                    shutil.rmtree(dirname)
            if not os.path.exists(dirname):
                self._download_to_dir(url, dirname)

    def _download_to_dir(self, url, dirname):
        print dirname
        cl = ["wget", url]
        subprocess.check_call(cl)
        cl = ["tar", "-xzvpf", os.path.basename(url)]
        subprocess.check_call(cl)
        os.rename(os.path.basename(dirname), dirname)
        os.remove(os.path.basename(url))

    @attr(speed=3)
    def IGNOREtest_3_full_pipeline(self):
        """Run full automated analysis pipeline with multiplexing.

        XXX Multiplexing not supporting in latest versions.
        """
        self._install_test_files(self.data_dir)
        with make_workdir() as workdir:
            cl = ["bcbio_nextgen.py",
                  get_post_process_yaml(self.data_dir, workdir),
                  os.path.join(self.data_dir, os.pardir, "110106_FC70BUKAAXX"),
                  os.path.join(self.data_dir, "run_info.yaml")]
            subprocess.check_call(cl)

    @attr(speed=3)
    def IGNOREtest_4_empty_fastq(self):
        """Handle analysis of empty fastq inputs from failed runs.

        XXX Multiplexing not supporting in latest versions.
        """
        with make_workdir() as workdir:
            cl = ["bcbio_nextgen.py",
                  get_post_process_yaml(self.data_dir, workdir),
                  os.path.join(self.data_dir, os.pardir, "110221_empty_FC12345AAXX"),
                  os.path.join(self.data_dir, "run_info-empty.yaml")]
            subprocess.check_call(cl)

    @attr(stranded=True)
    def test_2_stranded(self):
        """Run an RNA-seq analysis with TopHat and generate gene-level counts.
        """
        self._install_test_files(self.data_dir)
        with make_workdir() as workdir:
            cl = ["bcbio_nextgen.py",
                  get_post_process_yaml(self.data_dir, workdir),
                  os.path.join(self.data_dir, os.pardir, "test_stranded"),
                  os.path.join(self.data_dir, "run_info-stranded.yaml")]
            subprocess.check_call(cl)

    @attr(rnaseq=True)
    @attr(tophat=True)
    def test_2_rnaseq(self):
        """Run an RNA-seq analysis with TopHat and generate gene-level counts.
        """
        self._install_test_files(self.data_dir)
        with make_workdir() as workdir:
            cl = ["bcbio_nextgen.py",
                  get_post_process_yaml(self.data_dir, workdir),
                  os.path.join(self.data_dir, os.pardir, "110907_ERP000591"),
                  os.path.join(self.data_dir, "run_info-rnaseq.yaml")]
            subprocess.check_call(cl)

    @expected_failure
    @attr(fusion=True)
    def test_2_fusion(self):
        """Run an RNA-seq analysis and test fusion genes
        """
        self._install_test_files(self.data_dir)
        with make_workdir() as workdir:
            cl = ["bcbio_nextgen.py",
                  "/usr/local/share/bcbio-nextgen/galaxy/bcbio_system.yaml",
                  # get_post_process_yaml(self.data_dir, workdir),
                  os.path.join(self.data_dir, os.pardir, "test_fusion"),
                  os.path.join(self.data_dir, "run_info-fusion.yaml")]
            subprocess.check_call(cl)

    @attr(rnaseq=True)
    @attr(star=True)
    def test_2_star(self):
        """Run an RNA-seq analysis with STAR and generate gene-level counts.
        """
        self._install_test_files(self.data_dir)
        with make_workdir() as workdir:
            cl = ["bcbio_nextgen.py",
                  get_post_process_yaml(self.data_dir, workdir),
                  os.path.join(self.data_dir, os.pardir, "110907_ERP000591"),
                  os.path.join(self.data_dir, "run_info-star.yaml")]
            subprocess.check_call(cl)

    @attr(explant=True)
    def test_explant(self):
        """
        Run an explant RNA-seq analysis with TopHat and generate gene-level counts.
        """
        self._install_test_files(self.data_dir)
        with make_workdir() as workdir:
            cl = ["bcbio_nextgen.py",
                  get_post_process_yaml(self.data_dir, workdir),
                  os.path.join(self.data_dir, os.pardir, "1_explant"),
                  os.path.join(self.data_dir, "run_info-explant.yaml")]
            subprocess.check_call(cl)

    @attr(chipseq=True)
    def test_chipseq(self):
        """
        Run a chip-seq alignment with Bowtie2
        """
        self._install_test_files(self.data_dir)
        with make_workdir() as workdir:
            cl = ["bcbio_nextgen.py",
                  get_post_process_yaml(self.data_dir, workdir),
                  os.path.join(self.data_dir, os.pardir, "test_chipseq"),
                  os.path.join(self.data_dir, "run_info-chipseq.yaml")]
            subprocess.check_call(cl)

    @attr(speed=1)
    @attr(ensemble=True)
    def test_1_variantcall(self):
        """Test variant calling with GATK pipeline.
        """
        self._install_test_files(self.data_dir)
        with make_workdir() as workdir:
            cl = ["bcbio_nextgen.py",
                  get_post_process_yaml(self.data_dir, workdir),
                  os.path.join(self.data_dir, os.pardir, "100326_FC6107FAAXX"),
                  os.path.join(self.data_dir, "run_info-variantcall.yaml")]
            subprocess.check_call(cl)

    @attr(speed=1)
    @attr(devel=True)
    def test_5_bam(self):
        """Allow BAM files as input to pipeline.
        """
        self._install_test_files(self.data_dir)
        with make_workdir() as workdir:
            cl = ["bcbio_nextgen.py",
                  get_post_process_yaml(self.data_dir, workdir),
                  os.path.join(self.data_dir, os.pardir, "100326_FC6107FAAXX"),
                  os.path.join(self.data_dir, "run_info-bam.yaml")]
            subprocess.check_call(cl)

    @attr(speed=2)
    def test_6_bamclean(self):
        """Clean problem BAM input files that do not require alignment.
        """
        self._install_test_files(self.data_dir)
        with make_workdir() as workdir:
            cl = ["bcbio_nextgen.py",
                  get_post_process_yaml(self.data_dir, workdir),
                  os.path.join(self.data_dir, os.pardir, "100326_FC6107FAAXX"),
                  os.path.join(self.data_dir, "run_info-bamclean.yaml")]
            subprocess.check_call(cl)

    @attr(speed=2)
    @attr(cancer=True)
    @attr(cancermulti=True)
    def test_7_cancer(self):
        """Test paired tumor-normal calling using multiple calling approaches: MuTect, VarScan, FreeBayes.
        """
        self._install_test_files(self.data_dir)
        with make_workdir() as workdir:
            cl = ["bcbio_nextgen.py",
                  get_post_process_yaml(self.data_dir, workdir),
                  os.path.join(self.data_dir, os.pardir, "tcga_benchmark"),
                  os.path.join(self.data_dir, "run_info-cancer.yaml")]
            subprocess.check_call(cl)

    @attr(cancer=True)
    def test_7_cancer_nonormal(self):
        """Test cancer calling without normal samples or with normal VCF panels.
        """
        self._install_test_files(self.data_dir)
        with make_workdir() as workdir:
            cl = ["bcbio_nextgen.py",
                  get_post_process_yaml(self.data_dir, workdir),
                  os.path.join(self.data_dir, "run_info-cancer2.yaml")]
            subprocess.check_call(cl)

    @attr(speed=1)
    @attr(template=True)
    def test_8_template(self):
        """Create a project template from input files and metadata configuration.
        """
        self._install_test_files(self.data_dir)
        fc_dir = os.path.join(self.data_dir, os.pardir, "100326_FC6107FAAXX")
        with make_workdir() as workdir:
            cl = ["bcbio_nextgen.py", "-w", "template", "freebayes-variant",
                  os.path.join(fc_dir, "100326.csv"),
                  os.path.join(fc_dir, "7_100326_FC6107FAAXX_1_fastq.txt"),
                  os.path.join(fc_dir, "7_100326_FC6107FAAXX_2_fastq.txt"),
                  os.path.join(fc_dir, "8_100326_FC6107FAAXX.bam")]
            subprocess.check_call(cl)

    @attr(docker=True)
    def test_docker(self):
        """Run an analysis with code and tools inside a docker container.

        Requires https://github.com/chapmanb/bcbio-nextgen-vm
        """
        self._install_test_files(self.data_dir)
        with make_workdir() as workdir:
            cl = ["bcbio_vm.py", "run",
                  "--systemconfig=%s" % get_post_process_yaml(self.data_dir, workdir),
                  "--fcdir=%s" % os.path.join(self.data_dir, os.pardir, "100326_FC6107FAAXX"),
                  os.path.join(self.data_dir, "run_info-bam.yaml")]
            subprocess.check_call(cl)

    @attr(docker_ipython=True)
    def test_docker_ipython(self):
        """Run an analysis with code and tools inside a docker container, driven via IPython.

        Requires https://github.com/chapmanb/bcbio-nextgen-vm
        """
        self._install_test_files(self.data_dir)
        with make_workdir() as workdir:
            cl = ["bcbio_vm.py", "ipython",
                  "--systemconfig=%s" % get_post_process_yaml(self.data_dir, workdir),
                  "--fcdir=%s" % os.path.join(self.data_dir, os.pardir, "100326_FC6107FAAXX"),
                  os.path.join(self.data_dir, "run_info-bam.yaml"),
                  "lsf", "localrun"]
            subprocess.check_call(cl)
<|endoftext|>


<commit_before>from paddle.v2.fluid import framework as framework
from . import core
import collections

__all__ = ['append_backward']

def _rename_arg_(op_desc_list, old_name, new_name, begin_idx=None,
                 end_idx=None):
    if begin_idx is None:
        begin_idx = 0
    if end_idx is None:
        end_idx = len(op_desc_list)
    for i in range(begin_idx, end_idx):
        op_desc = op_desc_list[i]
        if isinstance(op_desc, tuple):
            op_desc = op_desc[0]
        op_desc.rename_input(old_name, new_name)
        op_desc.rename_output(old_name, new_name)

def _create_op_desc_(op_type, inputs, outputs, attrs):
    op_desc = core.OpDesc()
    op_desc.set_type(op_type)
    for para, args in inputs.iteritems():
        op_desc.set_input(para, args)
    for para, args in outputs.iteritems():
        op_desc.set_output(para, args)
    for name, val in attrs.iteritems():
        if isinstance(val, framework.Block):
            op_desc.set_block_attr(name, val.desc)
        else:
            op_desc.set_attr(name, val)
    return op_desc

def _infer_var_data_type_(var_name, block):
    grad_var = block.desc.find_var(var_name.encode("ascii"))
    fwd_name = _strip_grad_suffix_(var_name.encode("ascii"))
    if block.desc.has_var_recursive(fwd_name):
        fwd_var = block.desc.find_var_recursive(fwd_name.encode("ascii"))
        grad_var.set_dtype(fwd_var.dtype())
    else:
        grad_var.set_dtype(core.DataType.FP32)

def _all_in_set_(cands, s):
    for c in cands:
        if not c in s:
            return False
    return True

def _strip_grad_suffix_(name):
    pos = name.find(core.grad_var_suffix())
    return name[:pos] if pos != -1 else name

def _append_grad_suffix_(name):
    return name + core.grad_var_suffix()

def _addup_repetitive_outputs_(op_descs):
    # In backward part, an variable my be the output of more than one ops.
    # In this case, the variable should be the accumulation of all the outputs.
    # We adopt adding `sum_op`s to implement the accumulate.
    pending_sum_ops = []
    var_rename_count = collections.defaultdict(int)
    renamed_vars = collections.defaultdict(list)
    for idx, op_desc in enumerate(op_descs):
        for var_name in op_desc.input_arg_names():
            if len(renamed_vars[var_name]) > 1:
                pending_sum_ops.append(
                    (_create_op_desc_("sum", {"X": renamed_vars[var_name]},
                                      {"Out": [var_name]}, {}), idx))
                renamed_vars[var_name] = [var_name]
        for var_name in op_desc.output_arg_names():
            if var_name == core.empty_var_name(
            ) or var_name in op_desc.input_arg_names():
                # empty variable or inplace op
                continue
            if len(renamed_vars[var_name]) == 0:
                # it's the first time we get the variable
                renamed_vars[var_name] = [var_name]
            else:
                if len(renamed_vars[var_name]) == 1:
                    new_name = var_name + "@RENAME@" + \
                        str(var_rename_count[var_name])
                    var_rename_count[var_name] += 1
                    # rename original var_name
                    renamed_vars[var_name][0] = new_name
                    _rename_arg_(op_descs, var_name, new_name, 0, idx)
                    _rename_arg_(pending_sum_ops, var_name, new_name)

                new_name = var_name + "@RENAME@" + \
                    str(var_rename_count[var_name])
                var_rename_count[var_name] += 1
                op_desc.rename_output(var_name, new_name)
                renamed_vars[var_name].append(new_name)
    for var_name, inputs in renamed_vars.iteritems():
        if len(inputs) > 1:
            pending_sum_ops.append((_create_op_desc_(
                "sum", {"X": inputs}, {"Out": [var_name]}, {}), len(op_descs)))
    # sum_op descs are sorted according to their insert position
    for p in reversed(pending_sum_ops):
        op_descs.insert(p[1], p[0])

    return op_descs

def _remove_no_grad_branch_(op_descs, no_grad_set):
    # Remove ops whose outputs are all in no_grad_dict
    op_descs = filter(
        lambda op_desc: not _all_in_set_(op_desc.output_arg_names(), no_grad_set),
        op_descs)
    # Insert fill_zeros_like_op
    to_insert = []
    for idx, op_desc in enumerate(op_descs):
        for arg in op_desc.input_arg_names():
            if core.grad_var_suffix() in arg and arg in no_grad_set:
                to_insert.append((_create_op_desc_("fill_zeros_like", {
                    "X": [_strip_grad_suffix_(arg)]
                }, {"Y": [arg]}, {}), idx))

    map(lambda p: op_descs.insert(p[1], p[0]), reversed(to_insert))

    return op_descs

def _append_backward_ops_(target,
                          block,
                          target_block,
                          no_grad_dict,
                          grad_to_var,
                          callback=None):
    grad_op_descs = []
    program = block.program
    for op in reversed(block.ops):
        grad_sub_block_list = []
        # If the op has its own sub-block, deal with the sub-block first
        if op.has_attr("sub_block"):
            sub_block = program.block(op.block_attr("sub_block"))
            grad_sub_block = program.create_block(parent_idx=sub_block.idx)
            _append_backward_ops_(target, sub_block, grad_sub_block,
                                  no_grad_dict, grad_to_var, callback)
            grad_sub_block_list.append(grad_sub_block.desc)

        grad_op_desc, op_grad_to_var = core.get_grad_op_desc(
            op.desc, no_grad_dict[block.idx], grad_sub_block_list)
        grad_op_descs.extend(grad_op_desc)
        grad_to_var.update(op_grad_to_var)

    grad_op_descs = _addup_repetitive_outputs_(grad_op_descs)

    grad_op_descs = _remove_no_grad_branch_(grad_op_descs,
                                            no_grad_dict[block.idx])

    if target_block.idx == 0:
        grad_op_descs.insert(
            0,
            _create_op_desc_("fill_constant", {}, {
                "Out": [_append_grad_suffix_(target.name)]
            }, {"shape": [1],
                "value": 1.0,
                "dtype": target.dtype}))
    # append op_desc in grad_op_descs to target_block
    for op_desc in grad_op_descs:
        new_op_desc = target_block.desc.append_op()
        new_op_desc.copy_from(op_desc)

def _append_backward_vars_(block, start_op_idx, grad_to_var, grad_info_map):
    for op_idx in range(start_op_idx, block.desc.op_size()):
        op_desc = block.desc.op(op_idx)
        if op_desc.has_attr("sub_block"):
            sub_block = block.program.block(op_desc.block_attr("sub_block"))
            _append_backward_vars_(sub_block, 0, grad_to_var, grad_info_map)
        new_vars = set()
        # create new gradient variables
        for grad_var_name in op_desc.output_arg_names():
            grad_var_name = grad_var_name.encode("ascii")
            if block.desc.has_var_recursive(
                    grad_var_name) or grad_var_name == core.empty_var_name():
                continue
            block.desc.var(grad_var_name)
            new_vars.add(grad_var_name)
            if not grad_to_var.has_key(grad_var_name):
                continue
            grad_info_map[grad_to_var[grad_var_name]] = (grad_var_name, block)
        # infer_shape and infer_type
        op_desc.infer_var_type(block.desc)
        op_desc.infer_shape(block.desc)
        for arg in op_desc.output_arg_names():
            if arg in new_vars:
                _infer_var_data_type_(arg, block)

def append_backward(loss, parameter_list=None, no_grad_dict=None):
    """
    Create and add gradient Operators in BlockDesc to compute
    gradients of `loss` for parameters in parameter_list

    :param loss: an variable generated by cost function.
    :type loss: Variable
    :param no_grad_dict: variable that should not create gradient
    :type no_grad_dict: set
    :param parameter_list: parameters that need to compute gradient and
    update to optimize the lost.
    :type: list
    :return: list of (parameters, gradients) pair.
    :rtype: list[Variable]
    """
    assert isinstance(loss, framework.Variable)

    program = loss.block.program
    if no_grad_dict is None:
        no_grad_dict = dict()
        assert isinstance(program, framework.Program)
        for block in program.blocks:
            assert isinstance(block, framework.Block)
            block_no_grad_set = set()
            for var in block.vars.itervalues():
                assert isinstance(var, framework.Variable)
                if var.stop_gradient:
                    block_no_grad_set.add(_append_grad_suffix_(var.name))
            no_grad_dict[block.idx] = block_no_grad_set
    elif isinstance(no_grad_dict, set):
        no_grad_dict = {0: no_grad_dict}

    grad_info_map = dict()
    root_block = program.block(0)

    fwd_op_num = root_block.desc.op_size()
    current_block_idx = program.current_block_idx
    grad_to_var = dict()

    _append_backward_ops_(loss, root_block, root_block, no_grad_dict,
                          grad_to_var)
    _append_backward_vars_(root_block, fwd_op_num, grad_to_var, grad_info_map)

    program.current_block_idx = current_block_idx
    program.sync_with_cpp()

    if parameter_list is not None:
        parameters = parameter_list
    else:
        params = program.global_block().all_parameters()
        parameters = [param.name for param in params]
    params_and_grads = []
    for param in parameters:
        if param not in grad_info_map:
            raise ValueError("param %s is not in map" % param)
        grad_info = grad_info_map[param]
        grad_block = grad_info[1]
        if not grad_block.has_var(grad_info[0]):
            raise ValueError("grad block[{0}] did not have grad var {1}".format(
                grad_info[1], grad_info[0]))
        # Get the param var from the global block
        param_var = program.global_block().var(param)
        grad_var = grad_block.var(grad_info[0])
        if loss.block.has_var(grad_info[0]):
            params_and_grads.append((param_var, grad_var))
        else:
            params_and_grads.append((param_var, None))
    return params_and_grads
<commit_msg>fix errors<commit_after>from paddle.v2.fluid import framework as framework
from . import core
import collections

__all__ = ['append_backward']

def _rename_arg_(op_desc_list, old_name, new_name, begin_idx=None,
                 end_idx=None):
    if begin_idx is None:
        begin_idx = 0
    if end_idx is None:
        end_idx = len(op_desc_list)
    for i in range(begin_idx, end_idx):
        op_desc = op_desc_list[i]
        if isinstance(op_desc, tuple):
            op_desc = op_desc[0]
        op_desc.rename_input(old_name, new_name)
        op_desc.rename_output(old_name, new_name)

def _create_op_desc_(op_type, inputs, outputs, attrs):
    op_desc = core.OpDesc()
    op_desc.set_type(op_type)
    for para, args in inputs.iteritems():
        op_desc.set_input(para, args)
    for para, args in outputs.iteritems():
        op_desc.set_output(para, args)
    for name, val in attrs.iteritems():
        if isinstance(val, framework.Block):
            op_desc.set_block_attr(name, val.desc)
        else:
            op_desc.set_attr(name, val)
    return op_desc

def _infer_var_data_type_(var_name, block):
    grad_var = block.desc.find_var(var_name.encode("ascii"))
    fwd_name = _strip_grad_suffix_(var_name.encode("ascii"))
    if block.desc.has_var_recursive(fwd_name):
        fwd_var = block.desc.find_var_recursive(fwd_name.encode("ascii"))
        grad_var.set_dtype(fwd_var.dtype())
    else:
        grad_var.set_dtype(core.DataType.FP32)

def _all_in_set_(cands, s):
    for c in cands:
        if not c in s:
            return False
    return True

def _strip_grad_suffix_(name):
    pos = name.find(core.grad_var_suffix())
    return name[:pos] if pos != -1 else name

def _append_grad_suffix_(name):
    return name + core.grad_var_suffix()

def _addup_repetitive_outputs_(op_descs):
    # In backward part, an variable my be the output of more than one ops.
    # In this case, the variable should be the accumulation of all the outputs.
    # We adopt adding `sum_op`s to implement the accumulate.
    pending_sum_ops = []
    var_rename_count = collections.defaultdict(int)
    renamed_vars = collections.defaultdict(list)
    for idx, op_desc in enumerate(op_descs):
        for var_name in op_desc.input_arg_names():
            if len(renamed_vars[var_name]) > 1:
                pending_sum_ops.append(
                    (_create_op_desc_("sum", {"X": renamed_vars[var_name]},
                                      {"Out": [var_name]}, {}), idx))
                renamed_vars[var_name] = [var_name]
        for var_name in op_desc.output_arg_names():
            if var_name == core.empty_var_name(
            ) or var_name in op_desc.input_arg_names():
                # empty variable or inplace op
                continue
            if len(renamed_vars[var_name]) == 0:
                # it's the first time we get the variable
                renamed_vars[var_name] = [var_name]
            else:
                if len(renamed_vars[var_name]) == 1:
                    new_name = var_name + "@RENAME@" + \
                        str(var_rename_count[var_name])
                    var_rename_count[var_name] += 1
                    # rename original var_name
                    renamed_vars[var_name][0] = new_name
                    _rename_arg_(op_descs, var_name, new_name, 0, idx)
                    _rename_arg_(pending_sum_ops, var_name, new_name)

                new_name = var_name + "@RENAME@" + \
                    str(var_rename_count[var_name])
                var_rename_count[var_name] += 1
                op_desc.rename_output(var_name, new_name)
                renamed_vars[var_name].append(new_name)
    for var_name, inputs in renamed_vars.iteritems():
        if len(inputs) > 1:
            pending_sum_ops.append((_create_op_desc_(
                "sum", {"X": inputs}, {"Out": [var_name]}, {}), len(op_descs)))
    # sum_op descs are sorted according to their insert position
    for p in reversed(pending_sum_ops):
        op_descs.insert(p[1], p[0])

    return op_descs

def _remove_no_grad_branch_(op_descs, no_grad_set):
    # Remove ops whose outputs are all in no_grad_dict
    op_descs = filter(
        lambda op_desc: not _all_in_set_(op_desc.output_arg_names(), no_grad_set),
        op_descs)
    # Insert fill_zeros_like_op
    to_insert = []
    for idx, op_desc in enumerate(op_descs):
        for arg in op_desc.input_arg_names():
            if core.grad_var_suffix() in arg and arg in no_grad_set:
                to_insert.append((_create_op_desc_("fill_zeros_like", {
                    "X": [_strip_grad_suffix_(arg)]
                }, {"Y": [arg]}, {}), idx))

    map(lambda p: op_descs.insert(p[1], p[0]), reversed(to_insert))

    return op_descs

def _append_backward_ops_(target,
                          block,
                          target_block,
                          no_grad_dict,
                          grad_to_var,
                          callback=None):
    grad_op_descs = []
    program = block.program
    for op in reversed(block.ops):
        grad_sub_block_list = []
        # If the op has its own sub-block, deal with the sub-block first
        if op.has_attr("sub_block"):
            sub_block = program.block(op.block_attr("sub_block"))
            grad_sub_block = program.create_block(parent_idx=sub_block.idx)
            _append_backward_ops_(target, sub_block, grad_sub_block,
                                  no_grad_dict, grad_to_var, callback)
            grad_sub_block_list.append(grad_sub_block.desc)

        grad_op_desc, op_grad_to_var = core.get_grad_op_desc(
            op.desc, no_grad_dict[block.idx], grad_sub_block_list)
        grad_op_descs.extend(grad_op_desc)
        grad_to_var.update(op_grad_to_var)

    grad_op_descs = _addup_repetitive_outputs_(grad_op_descs)

    grad_op_descs = _remove_no_grad_branch_(grad_op_descs,
                                            no_grad_dict[block.idx])

    if target_block.idx == 0:
        grad_op_descs.insert(
            0,
            _create_op_desc_("fill_constant", {}, {
                "Out": [_append_grad_suffix_(target.name)]
            }, {"shape": [1],
                "value": 1.0,
                "dtype": target.dtype}))
    # append op_desc in grad_op_descs to target_block
    for op_desc in grad_op_descs:
        new_op_desc = target_block.desc.append_op()
        new_op_desc.copy_from(op_desc)

def _append_backward_vars_(block, start_op_idx, grad_to_var, grad_info_map):
    for op_idx in range(start_op_idx, block.desc.op_size()):
        op_desc = block.desc.op(op_idx)
        if op_desc.has_attr("sub_block"):
            sub_block = block.program.block(op_desc.block_attr("sub_block"))
            _append_backward_vars_(sub_block, 0, grad_to_var, grad_info_map)
        new_vars = set()
        # create new gradient variables
        for grad_var_name in op_desc.output_arg_names():
            grad_var_name = grad_var_name.encode("ascii")
            if block.desc.has_var_recursive(
                    grad_var_name) or grad_var_name == core.empty_var_name():
                continue
            block.desc.var(grad_var_name)
            new_vars.add(grad_var_name)
            if not grad_to_var.has_key(grad_var_name):
                continue
            grad_info_map[grad_to_var[grad_var_name]] = (grad_var_name, block)
        # infer_shape and infer_type
        op_desc.infer_var_type(block.desc)
        op_desc.infer_shape(block.desc)
        for arg in op_desc.output_arg_names():
            if arg in new_vars:
                _infer_var_data_type_(arg, block)

def append_backward(loss, parameter_list=None, no_grad_set=None):
    """
    Create and add gradient Operators in BlockDesc to compute
    gradients of `loss` for parameters in parameter_list

    :param loss: an variable generated by cost function.
    :type loss: Variable
    :param no_grad_dict: variable that should not create gradient
    :type no_grad_dict: set
    :param parameter_list: parameters that need to compute gradient and
    update to optimize the lost.
    :type: list
    :return: list of (parameters, gradients) pair.
    :rtype: list[Variable]
    """
    assert isinstance(loss, framework.Variable)

    program = loss.block.program
    no_grad_dict = dict()
    if no_grad_set is None:
        assert isinstance(program, framework.Program)
        for block in program.blocks:
            assert isinstance(block, framework.Block)
            block_no_grad_set = set()
            for var in block.vars.itervalues():
                assert isinstance(var, framework.Variable)
                if var.stop_gradient:
                    block_no_grad_set.add(_append_grad_suffix_(var.name))
            no_grad_dict[block.idx] = block_no_grad_set
    elif isinstance(no_grad_set, set):
        no_grad_dict = {0: no_grad_set}
    else:
        raise ValueError("'no_grad_set' should be a set or None.")

    grad_info_map = dict()
    root_block = program.block(0)

    fwd_op_num = root_block.desc.op_size()
    current_block_idx = program.current_block_idx
    grad_to_var = dict()

    _append_backward_ops_(loss, root_block, root_block, no_grad_dict,
                          grad_to_var)
    _append_backward_vars_(root_block, fwd_op_num, grad_to_var, grad_info_map)

    program.current_block_idx = current_block_idx
    program.sync_with_cpp()

    if parameter_list is not None:
        parameters = parameter_list
    else:
        params = program.global_block().all_parameters()
        parameters = [param.name for param in params]
    params_and_grads = []
    for param in parameters:
        if param not in grad_info_map:
            raise ValueError("param %s is not in map" % param)
        grad_info = grad_info_map[param]
        grad_block = grad_info[1]
        if not grad_block.has_var(grad_info[0]):
            raise ValueError("grad block[{0}] did not have grad var {1}".format(
                grad_info[1], grad_info[0]))
        # Get the param var from the global block
        param_var = program.global_block().var(param)
        grad_var = grad_block.var(grad_info[0])
        if loss.block.has_var(grad_info[0]):
            params_and_grads.append((param_var, grad_var))
        else:
            params_and_grads.append((param_var, None))
    return params_and_grads
<|endoftext|>


<commit_before>#!/usr/bin/python
# -*- coding: utf-8 -*-

import tweepy
import os

# provide your credentials
consumer_key = ""
consumer_secret = ""
access_token = ""
access_token_secret = ""

# your hashtag or search query and tweet language (empty = all languages)
hashtag = "#yourHashtag"
tweetLanguage = ""

# blacklisted users and words
userBlacklist = []
wordBlacklist = ["RT", u"♺"]

# build savepoint path + file
last_id_filename = "last_id_hashtag_%s" % hashtag.replace("#", "").split(" ")[0]
rt_bot_path = os.path.dirname(os.path.abspath(__file__))
last_id_file = os.path.join(rt_bot_path, last_id_filename)

# create bot
auth = tweepy.OAuthHandler(consumer_key, consumer_secret)
auth.set_access_token(access_token, access_token_secret)
api = tweepy.API(auth)

# retrieve last savepoint if available
try:
	with open(last_id_file, "r") as file:
		savepoint = file.read()
except IOError:
	savepoint = ""
	print "No savepoint found. Trying to get as many results as possible."

# search query
timelineIterator = tweepy.Cursor(api.search, q=hashtag, since_id=savepoint, lang=tweetLanguage).items()

# put everything into a list to be able to sort/filter
timeline = []
for status in timelineIterator:
	timeline.append(status)

try:
    last_tweet_id = timeline[0].id
except IndexError:
    last_tweet_id = savepoint

# filter @replies/blacklisted words & users out and reverse timeline
timeline = filter(lambda status: status.text[0] != "@", timeline)
timeline = filter(lambda status: not any(word in status.text.split() for word in wordBlacklist), timeline)
timeline = filter(lambda status: status.from_user not in userBlacklist, timeline)
timeline.reverse()

tw_counter = 0
err_counter = 0

# iterate the timeline and retweet
for status in timeline:
	try:
		print "(%(date)s) %(name)s: %(message)s\n" % \
			{ "date" : status.created_at,
			"name" : status.from_user.encode('utf-8'),
			"message" : status.text.encode('utf-8') }

		api.retweet(status.id)
		last_tweet_id = status.id
		tw_counter += 1
	except tweepy.error.TweepError:
		# just in case tweet got deleted in the meantime or already retweeted
		err_counter += 1
		continue

print "Finished. %d Tweets retweeted, %d errors occured." % (tw_counter, err_counter)

# write last retweeted tweet id to file
with open(last_id_file, "w") as file:
	file.write(str(last_tweet_id))

<commit_msg>Redundant line removed<commit_after>#!/usr/bin/python
# -*- coding: utf-8 -*-

import tweepy
import os

# provide your credentials
consumer_key = ""
consumer_secret = ""
access_token = ""
access_token_secret = ""

# your hashtag or search query and tweet language (empty = all languages)
hashtag = "#yourHashtag"
tweetLanguage = ""

# blacklisted users and words
userBlacklist = []
wordBlacklist = ["RT", u"♺"]

# build savepoint path + file
last_id_filename = "last_id_hashtag_%s" % hashtag.replace("#", "").split(" ")[0]
rt_bot_path = os.path.dirname(os.path.abspath(__file__))
last_id_file = os.path.join(rt_bot_path, last_id_filename)

# create bot
auth = tweepy.OAuthHandler(consumer_key, consumer_secret)
auth.set_access_token(access_token, access_token_secret)
api = tweepy.API(auth)

# retrieve last savepoint if available
try:
	with open(last_id_file, "r") as file:
		savepoint = file.read()
except IOError:
	savepoint = ""
	print "No savepoint found. Trying to get as many results as possible."

# search query
timelineIterator = tweepy.Cursor(api.search, q=hashtag, since_id=savepoint, lang=tweetLanguage).items()

# put everything into a list to be able to sort/filter
timeline = []
for status in timelineIterator:
	timeline.append(status)

try:
    last_tweet_id = timeline[0].id
except IndexError:
    last_tweet_id = savepoint

# filter @replies/blacklisted words & users out and reverse timeline
timeline = filter(lambda status: status.text[0] != "@", timeline)
timeline = filter(lambda status: not any(word in status.text.split() for word in wordBlacklist), timeline)
timeline = filter(lambda status: status.from_user not in userBlacklist, timeline)
timeline.reverse()

tw_counter = 0
err_counter = 0

# iterate the timeline and retweet
for status in timeline:
	try:
		print "(%(date)s) %(name)s: %(message)s\n" % \
			{ "date" : status.created_at,
			"name" : status.from_user.encode('utf-8'),
			"message" : status.text.encode('utf-8') }

		api.retweet(status.id)
		tw_counter += 1
	except tweepy.error.TweepError:
		# just in case tweet got deleted in the meantime or already retweeted
		err_counter += 1
		continue

print "Finished. %d Tweets retweeted, %d errors occured." % (tw_counter, err_counter)

# write last retweeted tweet id to file
with open(last_id_file, "w") as file:
	file.write(str(last_tweet_id))

<|endoftext|>