<commit_before>#!/usr/bin/env python

import argparse, datetime, getpass, os, sys, shutil, traceback
from fabric.api import run, cd, local, get, settings, lcd, put
from fabric_ssh_config import getSSHInfoForHost
from fabric.context_managers import shell_env
from fabric.utils import abort

#Login as user test, but build in a directory by real username
username = 'test'
builddir = "/tmp/" + getpass.getuser() + "Kits/buildtemp"
version = "UNKNOWN"
nativelibdir = "/nativelibs/obj"  #  ~test/libs/... usually
defaultlicensedays = 70 #default trial license length

################################################
# CHECKOUT CODE INTO A TEMP DIR
################################################

def checkoutCode(voltdbGit, proGit, rbmqExportGit, gitloc):
    global buildir
    # clean out the existing dir
    run("rm -rf " + builddir)
    # make the build dir again
    run("mkdir -p " + builddir)
    # change to it
    with cd(builddir):
        # do the checkouts, collect checkout errors on both community &
        # pro repos so user gets status on both checkouts
        message = ""
        run("git clone -q %s/voltdb.git" % gitloc)
        result = run("cd voltdb; git checkout %s" % voltdbGit, warn_only=True)
        if result.failed:
            message = "VoltDB checkout failed. Missing branch %s." % rbmqExportGit

        run("git clone -q %s/pro.git" % gitloc)
        result = run("cd pro; git checkout %s" % proGit, warn_only=True)
        if result.failed:
            message += "\nPro checkout failed. Missing branch %s." % rbmqExportGit

        #rabbitmq isn't mirrored internally, so don't use gitloc
        run("git clone -q git@github.com:VoltDB/export-rabbitmq.git")
        result = run("cd export-rabbitmq; git checkout %s" % rbmqExportGit, warn_only=True)
        # Probably ok to use master for export-rabbitmq.
        if result.failed:
            print "\nExport-rabbitmg branch %s checkout failed. Defaulting to master." % rbmqExportGit

        if len(message) > 0:
            abort(message)

        return run("cat voltdb/version.txt").strip()

################################################
# MAKE A RELEASE DIR
################################################

def makeReleaseDir(releaseDir):
    # handle the case where a release dir exists for this version
    if os.path.exists(releaseDir):
        shutil.rmtree(releaseDir)
    # create a release dir
    os.makedirs(releaseDir)
    print "Created dir: " + releaseDir

################################################
# BUILD THE COMMUNITY VERSION
################################################

def buildCommunity():
    if build_mac:
        packageMacLib="true"
    else:
        packageMacLib="false"
    with cd(builddir + "/voltdb"):
        run("pwd")
        run("git status")
        run("git describe --dirty")
        run("ant -Djmemcheck=NO_MEMCHECK -Dkitbuild=%s %s clean default dist" % (packageMacLib,  build_args))

################################################
# BUILD THE ENTERPRISE VERSION
################################################

def buildEnterprise():
    if build_mac:
        packageMacLib="true"
    else:
        packageMacLib="false"
    with cd(builddir + "/pro"):
        run("pwd")
        run("git status")
        run("git describe --dirty")
        run("VOLTCORE=../voltdb ant -f mmt.xml -Djmemcheck=NO_MEMCHECK -Dallowreplication=true -DallowDrActiveActive=true -Dlicensedays=%d -Dkitbuild=%s %s clean dist.pro" % (defaultlicensedays, packageMacLib, build_args))

################################################
# BUILD THE PRO VERSION
################################################

#
def packagePro(version):
    print "Making license"
    makeTrialLicense(days=defaultlicensedays, dr_and_xdcr=False, nodes=3)
    print "Repacking pro kit"
    with cd(builddir + "/pro/obj/pro"):
        run("mkdir pro_kit_staging")
    with cd(builddir + "/pro/obj/pro/pro_kit_staging"):
        run("tar xf ../voltdb-ent-%s.tar.gz" % version)
        run("mv voltdb-ent-%s voltdb-pro-%s" % (version, version))
        run("cp %s/pro/trial_*.xml voltdb-pro-%s/voltdb/license.xml" % (builddir, version))
        run("tar cvfz ../voltdb-pro-%s.tar.gz voltdb-pro-%s" % (version, version))

################################################
# BUILD THE RABBITMQ EXPORT CONNECTOR
################################################
#Build rabbit MQ Exporter
def buildRabbitMQExport(version, dist_type):
    # Paths to the final kit for unpacking/repacking with rmq export
    paths = {
        'community': builddir + "/voltdb/obj/release",
        'ent' : builddir + "/pro/obj/pro/"
        }
    # Untar
    with cd(paths[dist_type]):
        run ("pwd")
        run ("rm voltdb-%s-%s.tar.gz" % (dist_type, version))
        run ("mkdir -p restage")
        run ("tar xf voltdb-%s-%s.tar.gz -C restage" % (dist_type, version))

    # Build RabbitMQ export jar and put it into the untarred kit
    with cd(builddir + "/export-rabbitmq"):
        run("pwd")
        run("git status")
        run("git describe --dirty", warn_only=True)
        run("VOLTDIST=%s/restage/voltdb-%s-%s ant" % (paths[dist_type], dist_type, version))

    # Retar
    with cd(paths[dist_type]):
        run("pwd")
        run("tar -C restage -czf voltdb-%s-%s.tar.gz voltdb-%s-%s" % (dist_type, version, dist_type, version))
        run ("rm restage")

################################################
# MAKE AN ENTERPRISE TRIAL LICENSE
################################################

# Must be called after buildEnterprise has been done
def makeTrialLicense(days=30, dr_and_xdcr="true", nodes=12):
    with cd(builddir + "/pro/tools"):
        run("./make_trial_licenses.pl -t %d -H %d -W %s" % (days, nodes, dr_and_xdcr ))

################################################
# MAKE A SHA256 checksum
################################################

def makeSHA256SUM(version, type):
    with cd(builddir + "/pro/obj/pro"):
        kitname="voltdb-" +  type + "-" + version
        run("sha256sum -b %s.tar.gz > %s.SHA256SUM" % (kitname, kitname))

################################################
# MAKE AN JAR FILES NEEDED TO PUSH TO MAVEN
################################################

def makeMavenJars():
    with cd(builddir + "/voltdb"):
        run("VOLTCORE=../voltdb ant -f build-client.xml maven-jars")

################################################
# COPY FILES
################################################

def copyFilesToReleaseDir(releaseDir, version, type=None):
    print "Copying files to releaseDir"
    if type:
        typeString="-" + type
    else:
        typeString=""
    get("%s/pro/obj/pro/voltdb%s-%s.tar.gz" % (builddir, typeString, version),
        "%s/voltdb%s-%s.tar.gz" % (releaseDir, typeString, version))
    get("%s/pro/obj/pro/voltdb%s-%s.SHA256SUM" % (builddir, typeString, version),
        "%s/voltdb%s-%s.SHA256SUM" % (releaseDir, typeString, version))

def copyCommunityFilesToReleaseDir(releaseDir, version, operatingsys):
    get("%s/voltdb/obj/release/voltdb-community-%s.tar.gz" % (builddir, version),
        "%s/voltdb-community-%s.tar.gz" % (releaseDir, version))

    # add stripped symbols
    if operatingsys == "LINUX":
        os.makedirs(releaseDir + "/other")
        get("%s/voltdb/obj/release/voltdb-%s.sym" % (builddir, version),
            "%s/other/%s-voltdb-voltkv-%s.sym" % (releaseDir, operatingsys, version))

def copyTrialLicenseToReleaseDir(releaseDir):
    get("%s/pro/trial_*.xml" % (builddir),
        "%s/license.xml" % (releaseDir))

def copyMavenJarsToReleaseDir(releaseDir, version):
    #The .jars and upload file must be in a directory called voltdb - it is the projectname
    mavenProjectDir = releaseDir + "/mavenjars/voltdb"
    if not os.path.exists(mavenProjectDir):
        os.makedirs(mavenProjectDir)

    #Get the voltdbclient-n.n.jar from the recently built community build
    get("%s/voltdb/obj/release/dist-client-java/voltdb/voltdbclient-%s.jar" % (builddir, version),
        "%s/voltdbclient-%s.jar" % (mavenProjectDir, version))
    #Get the upload.gradle file
    get("%s/voltdb/tools/kit_tools/upload.gradle" % (builddir),
        "%s/upload.gradle" % (mavenProjectDir))
    #Get the src and javadoc .jar files
    get("%s/voltdb/obj/release/voltdbclient-%s-javadoc.jar" % (builddir, version),
        "%s/voltdbclient-%s-javadoc.jar" % (mavenProjectDir, version))
    get("%s/voltdb/obj/release/voltdbclient-%s-sources.jar" % (builddir, version),
        "%s/voltdbclient-%s-sources.jar" % (mavenProjectDir, version))

################################################
# CREATE CANDIDATE SYMLINKS
################################################

def createCandidateSysmlink(releaseDir):
    candidateDir =  os.getenv('HOME') + "/releases/candidate";
    local("rm -rf " + candidateDir)
    local("ln -s %s %s" % (releaseDir, candidateDir))

################################################
# BACKUP RELEASE DIR
################################################

def backupReleaseDir(releaseDir,archiveDir,version):
    if not os.path.exists(archiveDir):
        os.makedirs(archiveDir)
    # make a backup with the timstamp of the  build
    timestamp = datetime.datetime.now().strftime("%y%m%d-%H%M%S")
    local("tar -czf %s/%s-%s.tgz %s" \
          % (archiveDir, version, timestamp, releaseDir))

################################################
# REMOVE NATIVE LIBS FROM SHARED DIRECTORY
################################################

def rmNativeLibs():
    # local("ls -l ~" + username + nativelibdir)
    local("rm -rf ~" + username + nativelibdir)

################################################
# GET THE GIT TAGS OR SHAS TO BUILD FROM
################################################

parser = argparse.ArgumentParser(description = "Create a full kit. With no args, will do build of master")
parser.add_argument('voltdb_sha', nargs="?", default="master", help="voltdb repository commit, tag or branch" )
parser.add_argument('pro_sha', nargs="?", default="master", help="pro repository commit, tag or branch" )
parser.add_argument('rabbitmq_sha', nargs="?", default="master", help="rabbitmq repository commit, tag or branch" )
parser.add_argument('-g','--gitloc', default="git@github.com:VoltDB", help="Repository location. For example: /home/github-mirror")
parser.add_argument('--nomac', action='store_true', help="Don't build Mac OSX")
parser.add_argument('--nocommunity', action='store_true', help="Don't build community")
args = parser.parse_args()

proTreeish = args.pro_sha
voltdbTreeish = args.voltdb_sha
rbmqExportTreeish = args.rabbitmq_sha

print args

build_community = not args.nocommunity
build_mac = not args.nomac

#If anything is missing we're going to dump this in oneoffs dir.
build_all = build_community and build_mac
if voltdbTreeish != proTreeish or not build_all:
    oneOff = True
else:
    oneOff  = False

rmNativeLibs()

try:
    build_args = os.environ['VOLTDB_BUILD_ARGS']
except:
    build_args=""

print "Building with pro: %s and voltdb: %s" % (proTreeish, voltdbTreeish)

build_errors=False

versionCentos = "unknown"
versionMac = "unknown"
releaseDir = "unknown"

# get ssh config [key_filename, hostname]
CentosSSHInfo = getSSHInfoForHost("volt15a")
MacSSHInfo = getSSHInfoForHost("voltmini")
UbuntuSSHInfo = getSSHInfoForHost("volt12d")

# build community kit on the mini so that .so can be picked up for unified kit
if build_mac or build_community:
    try:
        with settings(user=username,host_string=MacSSHInfo[1],disable_known_hosts=True,key_filename=MacSSHInfo[0]):
            versionMac = checkoutCode(voltdbTreeish, proTreeish, rbmqExportTreeish, args.gitloc)
            buildCommunity()
    except Exception as e:
        print traceback.format_exc()
        print "Could not build MAC kit. Exception: " + str(e) + ", Type: " + str(type(e))
        build_errors=True

# build kits on 15f
try:
    with settings(user=username,host_string=CentosSSHInfo[1],disable_known_hosts=True,key_filename=CentosSSHInfo[0]):
        versionCentos = checkoutCode(voltdbTreeish, proTreeish, rbmqExportTreeish, args.gitloc)
        if build_mac:
            assert versionCentos == versionMac

        if oneOff:
            releaseDir = "%s/releases/one-offs/%s-%s-%s" % \
                (os.getenv('HOME'), versionCentos, voltdbTreeish, proTreeish)
        else:
            releaseDir = os.getenv('HOME') + "/releases/" + voltdbTreeish
        makeReleaseDir(releaseDir)
        print "VERSION: " + versionCentos
        if build_community:
            buildCommunity()
            buildRabbitMQExport(versionCentos, "community")
            copyCommunityFilesToReleaseDir(releaseDir, versionCentos, "LINUX")
        buildEnterprise()
        buildRabbitMQExport(versionCentos, "ent")
        makeSHA256SUM(versionCentos,"ent")
        copyFilesToReleaseDir(releaseDir, versionCentos, "ent")
        packagePro(versionCentos)
        makeSHA256SUM(versionCentos,"pro")
        copyFilesToReleaseDir(releaseDir, versionCentos, "pro")
        makeTrialLicense()
        copyTrialLicenseToReleaseDir(releaseDir)
        makeMavenJars()
        copyMavenJarsToReleaseDir(releaseDir, versionCentos)

except Exception as e:
    print traceback.format_exc()
    print "Could not build LINUX kit. Exception: " + str(e) + ", Type: " + str(type(e))
    build_errors=True

rmNativeLibs()      # cleanup imported native libs so not picked up unexpectedly by other builds

exit (build_errors)
#archiveDir = os.path.join(os.getenv('HOME'), "releases", "archive", voltdbTreeish, versionCentos)
#backupReleaseDir(releaseDir, archiveDir, versionCentos)
<commit_msg>ENG-13281. Cleanup.<commit_after>#!/usr/bin/env python

import argparse, datetime, getpass, os, sys, shutil, traceback
from fabric.api import run, cd, local, get, settings, lcd, put
from fabric_ssh_config import getSSHInfoForHost
from fabric.context_managers import shell_env
from fabric.utils import abort

#Login as user test, but build in a directory by real username
username = 'test'
builddir = "/tmp/" + getpass.getuser() + "Kits/buildtemp"
version = "UNKNOWN"
nativelibdir = "/nativelibs/obj"  #  ~test/libs/... usually
defaultlicensedays = 70 #default trial license length

################################################
# CHECKOUT CODE INTO A TEMP DIR
################################################

def checkoutCode(voltdbGit, proGit, rbmqExportGit, gitloc):
    global buildir
    # clean out the existing dir
    run("rm -rf " + builddir)
    # make the build dir again
    run("mkdir -p " + builddir)
    # change to it
    with cd(builddir):
        # do the checkouts, collect checkout errors on both community &
        # pro repos so user gets status on both checkouts
        message = ""
        run("git clone -q %s/voltdb.git" % gitloc)
        result = run("cd voltdb; git checkout %s" % voltdbGit, warn_only=True)
        if result.failed:
            message = "VoltDB checkout failed. Missing branch %s." % rbmqExportGit

        run("git clone -q %s/pro.git" % gitloc)
        result = run("cd pro; git checkout %s" % proGit, warn_only=True)
        if result.failed:
            message += "\nPro checkout failed. Missing branch %s." % rbmqExportGit

        #rabbitmq isn't mirrored internally, so don't use gitloc
        run("git clone -q git@github.com:VoltDB/export-rabbitmq.git")
        result = run("cd export-rabbitmq; git checkout %s" % rbmqExportGit, warn_only=True)
        # Probably ok to use master for export-rabbitmq.
        if result.failed:
            print "\nExport-rabbitmg branch %s checkout failed. Defaulting to master." % rbmqExportGit

        if len(message) > 0:
            abort(message)

        return run("cat voltdb/version.txt").strip()

################################################
# MAKE A RELEASE DIR
################################################

def makeReleaseDir(releaseDir):
    # handle the case where a release dir exists for this version
    if os.path.exists(releaseDir):
        shutil.rmtree(releaseDir)
    # create a release dir
    os.makedirs(releaseDir)
    print "Created dir: " + releaseDir

################################################
# BUILD THE COMMUNITY VERSION
################################################

def buildCommunity():
    if build_mac:
        packageMacLib="true"
    else:
        packageMacLib="false"
    with cd(builddir + "/voltdb"):
        run("pwd")
        run("git status")
        run("git describe --dirty")
        run("ant -Djmemcheck=NO_MEMCHECK -Dkitbuild=%s %s clean default dist" % (packageMacLib,  build_args))

################################################
# BUILD THE ENTERPRISE VERSION
################################################

def buildEnterprise():
    if build_mac:
        packageMacLib="true"
    else:
        packageMacLib="false"
    with cd(builddir + "/pro"):
        run("pwd")
        run("git status")
        run("git describe --dirty")
        run("VOLTCORE=../voltdb ant -f mmt.xml -Djmemcheck=NO_MEMCHECK -Dallowreplication=true -DallowDrActiveActive=true -Dlicensedays=%d -Dkitbuild=%s %s clean dist.pro" % (defaultlicensedays, packageMacLib, build_args))

################################################
# BUILD THE PRO VERSION
################################################

#
def packagePro(version):
    print "Making license"
    makeTrialLicense(days=defaultlicensedays, dr_and_xdcr=False, nodes=3)
    print "Repacking pro kit"
    with cd(builddir + "/pro/obj/pro"):
        run("mkdir pro_kit_staging")
    with cd(builddir + "/pro/obj/pro/pro_kit_staging"):
        run("tar xf ../voltdb-ent-%s.tar.gz" % version)
        run("mv voltdb-ent-%s voltdb-pro-%s" % (version, version))
        run("cp %s/pro/trial_*.xml voltdb-pro-%s/voltdb/license.xml" % (builddir, version))
        run("tar cvfz ../voltdb-pro-%s.tar.gz voltdb-pro-%s" % (version, version))

################################################
# BUILD THE RABBITMQ EXPORT CONNECTOR
################################################
#Build rabbit MQ Exporter
def buildRabbitMQExport(version, dist_type):
    # Paths to the final kit for unpacking/repacking with rmq export
    paths = {
        'community': builddir + "/voltdb/obj/release",
        'ent' : builddir + "/pro/obj/pro/"
        }
    # Untar
    with cd(paths[dist_type]):
        run ("pwd")
        run ("mkdir -p restage")
        run ("tar xf voltdb-%s-%s.tar.gz -C restage" % (dist_type, version))
        run ("rm voltdb-%s-%s.tar.gz" % (dist_type, version))

    # Build RabbitMQ export jar and put it into the untarred kit
    with cd(builddir + "/export-rabbitmq"):
        run("pwd")
        run("git status")
        run("git describe --dirty", warn_only=True)
        run("VOLTDIST=%s/restage/voltdb-%s-%s ant" % (paths[dist_type], dist_type, version))

    # Retar
    with cd(paths[dist_type]):
        run("pwd")
        run("tar -C restage -czf voltdb-%s-%s.tar.gz voltdb-%s-%s" % (dist_type, version, dist_type, version))
        run ("rm restage")

################################################
# MAKE AN ENTERPRISE TRIAL LICENSE
################################################

# Must be called after buildEnterprise has been done
def makeTrialLicense(days=30, dr_and_xdcr="true", nodes=12):
    with cd(builddir + "/pro/tools"):
        run("./make_trial_licenses.pl -t %d -H %d -W %s" % (days, nodes, dr_and_xdcr ))

################################################
# MAKE A SHA256 checksum
################################################

def makeSHA256SUM(version, type):
    with cd(builddir + "/pro/obj/pro"):
        kitname="voltdb-" +  type + "-" + version
        run("sha256sum -b %s.tar.gz > %s.SHA256SUM" % (kitname, kitname))

################################################
# MAKE AN JAR FILES NEEDED TO PUSH TO MAVEN
################################################

def makeMavenJars():
    with cd(builddir + "/voltdb"):
        run("VOLTCORE=../voltdb ant -f build-client.xml maven-jars")

################################################
# COPY FILES
################################################

def copyFilesToReleaseDir(releaseDir, version, type=None):
    print "Copying files to releaseDir"
    if type:
        typeString="-" + type
    else:
        typeString=""
    get("%s/pro/obj/pro/voltdb%s-%s.tar.gz" % (builddir, typeString, version),
        "%s/voltdb%s-%s.tar.gz" % (releaseDir, typeString, version))
    get("%s/pro/obj/pro/voltdb%s-%s.SHA256SUM" % (builddir, typeString, version),
        "%s/voltdb%s-%s.SHA256SUM" % (releaseDir, typeString, version))

def copyCommunityFilesToReleaseDir(releaseDir, version, operatingsys):
    get("%s/voltdb/obj/release/voltdb-community-%s.tar.gz" % (builddir, version),
        "%s/voltdb-community-%s.tar.gz" % (releaseDir, version))

    # add stripped symbols
    if operatingsys == "LINUX":
        os.makedirs(releaseDir + "/other")
        get("%s/voltdb/obj/release/voltdb-%s.sym" % (builddir, version),
            "%s/other/%s-voltdb-voltkv-%s.sym" % (releaseDir, operatingsys, version))

def copyTrialLicenseToReleaseDir(releaseDir):
    get("%s/pro/trial_*.xml" % (builddir),
        "%s/license.xml" % (releaseDir))

def copyMavenJarsToReleaseDir(releaseDir, version):
    #The .jars and upload file must be in a directory called voltdb - it is the projectname
    mavenProjectDir = releaseDir + "/mavenjars/voltdb"
    if not os.path.exists(mavenProjectDir):
        os.makedirs(mavenProjectDir)

    #Get the voltdbclient-n.n.jar from the recently built community build
    get("%s/voltdb/obj/release/dist-client-java/voltdb/voltdbclient-%s.jar" % (builddir, version),
        "%s/voltdbclient-%s.jar" % (mavenProjectDir, version))
    #Get the upload.gradle file
    get("%s/voltdb/tools/kit_tools/upload.gradle" % (builddir),
        "%s/upload.gradle" % (mavenProjectDir))
    #Get the src and javadoc .jar files
    get("%s/voltdb/obj/release/voltdbclient-%s-javadoc.jar" % (builddir, version),
        "%s/voltdbclient-%s-javadoc.jar" % (mavenProjectDir, version))
    get("%s/voltdb/obj/release/voltdbclient-%s-sources.jar" % (builddir, version),
        "%s/voltdbclient-%s-sources.jar" % (mavenProjectDir, version))

################################################
# CREATE CANDIDATE SYMLINKS
################################################

def createCandidateSysmlink(releaseDir):
    candidateDir =  os.getenv('HOME') + "/releases/candidate";
    local("rm -rf " + candidateDir)
    local("ln -s %s %s" % (releaseDir, candidateDir))

################################################
# BACKUP RELEASE DIR
################################################

def backupReleaseDir(releaseDir,archiveDir,version):
    if not os.path.exists(archiveDir):
        os.makedirs(archiveDir)
    # make a backup with the timstamp of the  build
    timestamp = datetime.datetime.now().strftime("%y%m%d-%H%M%S")
    local("tar -czf %s/%s-%s.tgz %s" \
          % (archiveDir, version, timestamp, releaseDir))

################################################
# REMOVE NATIVE LIBS FROM SHARED DIRECTORY
################################################

def rmNativeLibs():
    # local("ls -l ~" + username + nativelibdir)
    local("rm -rf ~" + username + nativelibdir)

################################################
# GET THE GIT TAGS OR SHAS TO BUILD FROM
################################################

parser = argparse.ArgumentParser(description = "Create a full kit. With no args, will do build of master")
parser.add_argument('voltdb_sha', nargs="?", default="master", help="voltdb repository commit, tag or branch" )
parser.add_argument('pro_sha', nargs="?", default="master", help="pro repository commit, tag or branch" )
parser.add_argument('rabbitmq_sha', nargs="?", default="master", help="rabbitmq repository commit, tag or branch" )
parser.add_argument('-g','--gitloc', default="git@github.com:VoltDB", help="Repository location. For example: /home/github-mirror")
parser.add_argument('--nomac', action='store_true', help="Don't build Mac OSX")
parser.add_argument('--nocommunity', action='store_true', help="Don't build community")
args = parser.parse_args()

proTreeish = args.pro_sha
voltdbTreeish = args.voltdb_sha
rbmqExportTreeish = args.rabbitmq_sha

print args

build_community = not args.nocommunity
build_mac = not args.nomac

#If anything is missing we're going to dump this in oneoffs dir.
build_all = build_community and build_mac
if voltdbTreeish != proTreeish or not build_all:
    oneOff = True
else:
    oneOff  = False

rmNativeLibs()

try:
    build_args = os.environ['VOLTDB_BUILD_ARGS']
except:
    build_args=""

print "Building with pro: %s and voltdb: %s" % (proTreeish, voltdbTreeish)

build_errors=False

versionCentos = "unknown"
versionMac = "unknown"
releaseDir = "unknown"

# get ssh config [key_filename, hostname]
CentosSSHInfo = getSSHInfoForHost("volt15a")
MacSSHInfo = getSSHInfoForHost("voltmini")
UbuntuSSHInfo = getSSHInfoForHost("volt12d")

# build community kit on the mini so that .so can be picked up for unified kit
if build_mac or build_community:
    try:
        with settings(user=username,host_string=MacSSHInfo[1],disable_known_hosts=True,key_filename=MacSSHInfo[0]):
            versionMac = checkoutCode(voltdbTreeish, proTreeish, rbmqExportTreeish, args.gitloc)
            buildCommunity()
    except Exception as e:
        print traceback.format_exc()
        print "Could not build MAC kit. Exception: " + str(e) + ", Type: " + str(type(e))
        build_errors=True

# build kits on 15f
try:
    with settings(user=username,host_string=CentosSSHInfo[1],disable_known_hosts=True,key_filename=CentosSSHInfo[0]):
        versionCentos = checkoutCode(voltdbTreeish, proTreeish, rbmqExportTreeish, args.gitloc)
        if build_mac:
            assert versionCentos == versionMac

        if oneOff:
            releaseDir = "%s/releases/one-offs/%s-%s-%s" % \
                (os.getenv('HOME'), versionCentos, voltdbTreeish, proTreeish)
        else:
            releaseDir = os.getenv('HOME') + "/releases/" + voltdbTreeish
        makeReleaseDir(releaseDir)
        print "VERSION: " + versionCentos
        if build_community:
            buildCommunity()
            buildRabbitMQExport(versionCentos, "community")
            copyCommunityFilesToReleaseDir(releaseDir, versionCentos, "LINUX")
        buildEnterprise()
        buildRabbitMQExport(versionCentos, "ent")
        makeSHA256SUM(versionCentos,"ent")
        copyFilesToReleaseDir(releaseDir, versionCentos, "ent")
        packagePro(versionCentos)
        makeSHA256SUM(versionCentos,"pro")
        copyFilesToReleaseDir(releaseDir, versionCentos, "pro")
        makeTrialLicense()
        copyTrialLicenseToReleaseDir(releaseDir)
        makeMavenJars()
        copyMavenJarsToReleaseDir(releaseDir, versionCentos)

except Exception as e:
    print traceback.format_exc()
    print "Could not build LINUX kit. Exception: " + str(e) + ", Type: " + str(type(e))
    build_errors=True

rmNativeLibs()      # cleanup imported native libs so not picked up unexpectedly by other builds

exit (build_errors)
#archiveDir = os.path.join(os.getenv('HOME'), "releases", "archive", voltdbTreeish, versionCentos)
#backupReleaseDir(releaseDir, archiveDir, versionCentos)
<|endoftext|>


<commit_before># Copyright 2013-2021 Lawrence Livermore National Security, LLC and other
# Spack Project Developers. See the top-level COPYRIGHT file for details.
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)

from spack import *

class PySix(PythonPackage):
    """Python 2 and 3 compatibility utilities."""

    pypi = "six/six-1.11.0.tar.gz"

    version('1.14.0', sha256='236bdbdce46e6e6a3d61a337c0f8b763ca1e8717c03b369e87a7ec7ce1319c0a')
    version('1.12.0', sha256='d16a0141ec1a18405cd4ce8b4613101da75da0e9a7aec5bdd4fa804d0e0eba73')
    version('1.11.0', sha256='70e8a77beed4562e7f14fe23a786b54f6296e34344c23bc42f07b15018ff98e9')
    version('1.10.0', sha256='105f8d68616f8248e24bf0e9372ef04d3cc10104f1980f54d57b2ce73a5ad56a')
    version('1.9.0',  sha256='e24052411fc4fbd1f672635537c3fc2330d9481b18c0317695b46259512c91d5')
    version('1.8.0',  sha256='047bbbba41bac37c444c75ddfdf0573dd6e2f1fbd824e6247bb26fa7d8fa3830')

    extends('python', ignore=r'bin/pytest')
    depends_on('py-setuptools', type='build')
<commit_msg>py-six: 1.15.0 (#20710)<commit_after># Copyright 2013-2021 Lawrence Livermore National Security, LLC and other
# Spack Project Developers. See the top-level COPYRIGHT file for details.
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)

from spack import *

class PySix(PythonPackage):
    """Python 2 and 3 compatibility utilities."""

    pypi = "six/six-1.11.0.tar.gz"

    version('1.15.0', sha256='30639c035cdb23534cd4aa2dd52c3bf48f06e5f4a941509c8bafd8ce11080259')
    version('1.14.0', sha256='236bdbdce46e6e6a3d61a337c0f8b763ca1e8717c03b369e87a7ec7ce1319c0a')
    version('1.12.0', sha256='d16a0141ec1a18405cd4ce8b4613101da75da0e9a7aec5bdd4fa804d0e0eba73')
    version('1.11.0', sha256='70e8a77beed4562e7f14fe23a786b54f6296e34344c23bc42f07b15018ff98e9')
    version('1.10.0', sha256='105f8d68616f8248e24bf0e9372ef04d3cc10104f1980f54d57b2ce73a5ad56a')
    version('1.9.0',  sha256='e24052411fc4fbd1f672635537c3fc2330d9481b18c0317695b46259512c91d5')
    version('1.8.0',  sha256='047bbbba41bac37c444c75ddfdf0573dd6e2f1fbd824e6247bb26fa7d8fa3830')

    extends('python', ignore=r'bin/pytest')
    depends_on('python@2.7:2.8,3.3:', type=('build', 'run'))
    depends_on('py-setuptools', type='build')
<|endoftext|>


<commit_before>#!/usr/bin/env python

# Delete all snapshots of a given ZFS filesystem but **NOT** of descendant filesystems
# Or -- if filesystem='90d', delete all snapshots of all filesystems whose name ends in "--90d".

import sys, time
from subprocess import Popen, PIPE

def cmd(v):
    t = time.time()
    print ' '.join(v),
    sys.stdout.flush()
    out = Popen(v,stdin=PIPE, stdout=PIPE, stderr=PIPE, shell=False)
    x = out.stdout.read()
    y = out.stderr.read()
    e = out.wait()
    if e:
        raise RuntimeError(y)
    print "    (%.2f seconds)"%(time.time()-t)
    return x

def delete_snapshots(filesystem):

    if filesystem == '90d':
        print "deleting all snapshots of any filesystem in any pool ending in --90d"
        x = cmd(['zfs', 'list', '-H', '-r', '-t', 'snapshot'])

        # take only those ending in --90d
        lines = [t for t in x.splitlines() if t.endswith('--90d')]

    else:
        print "deleting snapshots of filesystem %s"%filesystem
        x = cmd(['zfs', 'list', '-H', '-r', '-t', 'snapshot', filesystem])

        # get rid of descendant filesystems in list.
        lines = [t for t in x.splitlines() if filesystem+"@" in t]

    total = len(lines)
    print "%s snapshots to delete"%total

    i = 0
    for a in lines:
        if a:
            snapshot = a.split()[0]
            print snapshot
            #cmd(['zfs', 'destroy', snapshot])
            i += 1
            print "%s/%s"%(i,total)

for filesystem in sys.argv[1:]:
    delete_snapshots(filesystem)

<commit_msg>code that is active for deleting 90d snapshots<commit_after>#!/usr/bin/env python

# Delete all snapshots of a given ZFS filesystem but **NOT** of descendant filesystems
# Or -- if filesystem='90d', delete all snapshots of all filesystems whose name ends in "--90d".

import sys, time
from subprocess import Popen, PIPE

def cmd(v):
    t = time.time()
    print ' '.join(v),
    sys.stdout.flush()
    out = Popen(v,stdin=PIPE, stdout=PIPE, stderr=PIPE, shell=False)
    x = out.stdout.read()
    y = out.stderr.read()
    e = out.wait()
    if e:
        raise RuntimeError(y)
    print "    (%.2f seconds)"%(time.time()-t)
    return x

def delete_snapshots(filesystem):

    if filesystem == '90d':
        print "deleting all snapshots of any filesystem in any pool that contain '--90d\\t'"
        x = cmd(['zfs', 'list', '-H', '-r', '-t', 'snapshot'])

        # take only those ending in --90d
        lines = [t for t in x.splitlines() if '--90d\t'in t]

    else:
        print "deleting snapshots of filesystem %s"%filesystem
        x = cmd(['zfs', 'list', '-H', '-r', '-t', 'snapshot', filesystem])

        # get rid of descendant filesystems in list.
        lines = [t for t in x.splitlines() if filesystem+"@" in t]

    total = len(lines)
    print "%s snapshots to delete"%total

    i = 0
    for a in lines:
        if a:
            snapshot = a.split()[0]
            print snapshot
            cmd(['zfs', 'destroy', snapshot])
            i += 1
            print "%s/%s"%(i,total)

for filesystem in sys.argv[1:]:
    delete_snapshots(filesystem)

<|endoftext|>


<commit_before># Copyright 2013-2019 Lawrence Livermore National Security, LLC and other
# Spack Project Developers. See the top-level COPYRIGHT file for details.
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)

from spack import *

class Skopeo(MakefilePackage):
    """skopeo is a command line utility that performs various operations on
    container images and image repositories."""

    homepage = "https://github.com/containers/skopeo"
    url      = "https://github.com/containers/skopeo/archive/v0.1.39.tar.gz"

    version('0.1.39', sha256='e9d70f7f7b891675a816f06a22df0490285ad20eefbd91f5da69ca12f56c29f2')
    version('0.1.38', sha256='104ceb9c582dc5c3a49dd1752c4c326bba03f2f801596f089372e831f48ed705')
    version('0.1.37', sha256='49c0c1b2c2f32422d3230f827ae405fc554fb34af41a54e59b2121ac1500505d')
    version('0.1.36', sha256='42f9b0bf53ae44bc294be400e2c5259f977ffa4d5dbac3576b0b5e23d59791fd')

    depends_on('go')
    depends_on('gpgme')
    depends_on('libassuan')
    depends_on('libgpg-error')

    def edit(self, spec, prefix):
        grep = which('grep')
        files = grep('-lR', '/etc/containers/', 'vendor', output=str,
                     env={'PATH': '/usr/bin:/bin:/usr/sbin:/sbin'})

        for f in files.splitlines():
            edit = FileFilter(f)
            edit.filter('/etc/containers/', '{0}/etc/containers/'.
                        format(prefix))

    def build(self, spec, prefix):
        make('binary-local')

    def install(self, spec, prefix):
        make('binary-local', 'install',
             'DESTDIR={0}'.format(prefix),
             'PREFIX={0}'.format(prefix))
<commit_msg>Fix skopeo dependencies (#12425)<commit_after># Copyright 2013-2019 Lawrence Livermore National Security, LLC and other
# Spack Project Developers. See the top-level COPYRIGHT file for details.
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)

from spack import *

class Skopeo(MakefilePackage):
    """skopeo is a command line utility that performs various operations on
    container images and image repositories."""

    homepage = "https://github.com/containers/skopeo"
    url      = "https://github.com/containers/skopeo/archive/v0.1.39.tar.gz"

    version('0.1.39', sha256='e9d70f7f7b891675a816f06a22df0490285ad20eefbd91f5da69ca12f56c29f2')
    version('0.1.38', sha256='104ceb9c582dc5c3a49dd1752c4c326bba03f2f801596f089372e831f48ed705')
    version('0.1.37', sha256='49c0c1b2c2f32422d3230f827ae405fc554fb34af41a54e59b2121ac1500505d')
    version('0.1.36', sha256='42f9b0bf53ae44bc294be400e2c5259f977ffa4d5dbac3576b0b5e23d59791fd')

    depends_on('go')
    depends_on('go-md2man', type='build')
    depends_on('gpgme')
    depends_on('libassuan')
    depends_on('libgpg-error')
    depends_on('lvm2')

    def edit(self, spec, prefix):
        grep = which('grep')
        files = grep('-lR', '/etc/containers/', 'vendor', output=str,
                     env={'PATH': '/usr/bin:/bin:/usr/sbin:/sbin'})

        for f in files.splitlines():
            edit = FileFilter(f)
            edit.filter('/etc/containers/', '{0}/etc/containers/'.
                        format(prefix))

    def build(self, spec, prefix):
        make('binary-local')

    def install(self, spec, prefix):
        make('binary-local', 'install',
             'DESTDIR={0}'.format(prefix),
             'PREFIX={0}'.format(prefix))
<|endoftext|>


<commit_before>##############################################################################
# Copyright (c) 2013-2017, Lawrence Livermore National Security, LLC.
# Produced at the Lawrence Livermore National Laboratory.
#
# This file is part of Spack.
# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved.
# LLNL-CODE-647188
#
# For details, see https://github.com/spack/spack
# Please also see the NOTICE and LICENSE files for our notice and the LGPL.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License (as
# published by the Free Software Foundation) version 2.1, February 1999.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and
# conditions of the GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
##############################################################################
from spack import *
from spack import architecture

class Sqlite(AutotoolsPackage):
    """SQLite3 is an SQL database engine in a C library. Programs that
       link the SQLite3 library can have SQL database access without
       running a separate RDBMS process.
    """
    homepage = "www.sqlite.org"

    version('3.21.0', '7913de4c3126ba3c24689cb7a199ea31',
            url='https://www.sqlite.org/2017/sqlite-autoconf-3210000.tar.gz')
    version('3.20.0', 'e262a28b73cc330e7e83520c8ce14e4d',
            url='https://www.sqlite.org/2017/sqlite-autoconf-3200000.tar.gz')
    version('3.18.0', 'a6687a8ae1f66abc8df739aeadecfd0c',
            url='https://www.sqlite.org/2017/sqlite-autoconf-3180000.tar.gz')
    version('3.8.10.2', 'a18bfc015cd49a1e7a961b7b77bc3b37',
            url='https://www.sqlite.org/2015/sqlite-autoconf-3081002.tar.gz')
    version('3.8.5', '0544ef6d7afd8ca797935ccc2685a9ed',
            url='https://www.sqlite.org/2014/sqlite-autoconf-3080500.tar.gz')

    depends_on('readline')

    # On some platforms (e.g., PPC) the include chain includes termios.h which
    # defines a macro B0. Sqlite has a shell.c source file that declares a
    # variable named B0 and will fail to compile when the macro is found. The
    # following patch undefines the macro in shell.c
    patch('sqlite_b0.patch', when='@3.18.0:3.21.0')

    # Starting version 3.17.0, SQLite uses compiler built-ins
    # __builtin_sub_overflow(), __builtin_add_overflow(), and
    # __builtin_mul_overflow(), which are not supported by Intel compiler.
    # Starting version 3.21.0 SQLite doesn't use the built-ins if Intel
    # compiler is used.
    patch('remove_overflow_builtins.patch', when='@3.17.0:3.20%intel')

    def get_arch(self):
        arch = architecture.Arch()
        arch.platform = architecture.platform()
        return str(arch.platform.target('default_target'))

    def configure_args(self):
        args = []

        if self.get_arch() == 'ppc64le':
            args.append('--build=powerpc64le-redhat-linux-gnu')

        return args
<commit_msg>Add sqlite 3.22.0 (#7335)<commit_after>##############################################################################
# Copyright (c) 2013-2017, Lawrence Livermore National Security, LLC.
# Produced at the Lawrence Livermore National Laboratory.
#
# This file is part of Spack.
# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved.
# LLNL-CODE-647188
#
# For details, see https://github.com/spack/spack
# Please also see the NOTICE and LICENSE files for our notice and the LGPL.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License (as
# published by the Free Software Foundation) version 2.1, February 1999.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and
# conditions of the GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
##############################################################################
from spack import *
from spack import architecture

class Sqlite(AutotoolsPackage):
    """SQLite3 is an SQL database engine in a C library. Programs that
       link the SQLite3 library can have SQL database access without
       running a separate RDBMS process.
    """
    homepage = "www.sqlite.org"

    version('3.22.0', '2fb24ec12001926d5209d2da90d252b9825366ac',
            url='https://www.sqlite.org/2018/sqlite-autoconf-3220000.tar.gz')
    version('3.21.0', '7913de4c3126ba3c24689cb7a199ea31',
            url='https://www.sqlite.org/2017/sqlite-autoconf-3210000.tar.gz')
    version('3.20.0', 'e262a28b73cc330e7e83520c8ce14e4d',
            url='https://www.sqlite.org/2017/sqlite-autoconf-3200000.tar.gz')
    version('3.18.0', 'a6687a8ae1f66abc8df739aeadecfd0c',
            url='https://www.sqlite.org/2017/sqlite-autoconf-3180000.tar.gz')
    version('3.8.10.2', 'a18bfc015cd49a1e7a961b7b77bc3b37',
            url='https://www.sqlite.org/2015/sqlite-autoconf-3081002.tar.gz')
    version('3.8.5', '0544ef6d7afd8ca797935ccc2685a9ed',
            url='https://www.sqlite.org/2014/sqlite-autoconf-3080500.tar.gz')

    depends_on('readline')

    # On some platforms (e.g., PPC) the include chain includes termios.h which
    # defines a macro B0. Sqlite has a shell.c source file that declares a
    # variable named B0 and will fail to compile when the macro is found. The
    # following patch undefines the macro in shell.c
    patch('sqlite_b0.patch', when='@3.18.0:3.21.0')

    # Starting version 3.17.0, SQLite uses compiler built-ins
    # __builtin_sub_overflow(), __builtin_add_overflow(), and
    # __builtin_mul_overflow(), which are not supported by Intel compiler.
    # Starting version 3.21.0 SQLite doesn't use the built-ins if Intel
    # compiler is used.
    patch('remove_overflow_builtins.patch', when='@3.17.0:3.20%intel')

    def get_arch(self):
        arch = architecture.Arch()
        arch.platform = architecture.platform()
        return str(arch.platform.target('default_target'))

    def configure_args(self):
        args = []

        if self.get_arch() == 'ppc64le':
            args.append('--build=powerpc64le-redhat-linux-gnu')

        return args
<|endoftext|>


<commit_before>try:
    from zope.interface import Interface, Attribute
except ImportError:
    class Interface(object):
        """A dummy interface base class."""

    class Attribute(object):
        """A dummy attribute implementation."""
        def __init__(self, doc):
            self.doc = doc

_marker = object()

class IComponent(Interface):
    """Component is the base object for calendar, Event and the other
    components defined in RFC 2445.

    A component is like a dictionary with extra methods and attributes.

    """

    # MANIPULATORS

    def __setitem__(name, value):
        """Set a property.

        name - case insensitive name
        value - value of the property to set. This can be either a single
        item or a list.

        Some iCalendar properties are set INLINE; these properties
        have multiple values on one property line in the iCalendar
        representation.  The list can be supplied as a comma separated
        string to __setitem__. If special iCalendar characters exist in
        an entry, such as the colon (:) and (,), that comma-separated
        entry needs to be quoted with double quotes. For example:

        'foo, bar, "baz:hoi"'

        See also set_inline() for an easier way to deal with this case.

        """

    def set_inline(name, values, encode=1):
        """Set list of INLINE values for property.

        Converts a list of values into valid iCalendar comma seperated
        string and sets value to that.

        name - case insensitive name of property
        values - list of values to set
        encode - if True, encode Python values as iCalendar types first.

        """

    def add(name, value):
        """Add a property. Can be called multiple times to set a list.

        name - case insensitive name
        value - value of property to set or add to list for this property.

        """

    def add_component(component):
        """Add a nested subcomponent to this component."""

    # static method, can be called on class directly
    def from_ical(st, multiple=False):
        """Populates the component recursively from a iCalendar string.

        Reads the iCalendar string and constructs components and
        subcomponents out of it.

        """

    # ACCESSORS
    def __getitem__(name):
        """Get a property

        name - case insensitive name

        Returns an iCalendar property object such as vText.

        """

    def decoded(name, default=_marker):
        """Get a property as a python object.

        name - case insensitive name
        default - optional argument. If supplied, will use this if
        name cannot be found. If not supplied, decoded will raise a
        KeyError if name cannot be found.

        Returns python object (such as unicode string, datetime, etc).

        """

    def get_inline(name, decode=1):
        """Get list of INLINE values from property.

        name - case insensitive name
        decode - decode to Python objects.

        Returns list of python objects.

        """

    def to_ical():
        """Render the component in the RFC 2445 (iCalendar) format.

        Returns a string in RFC 2445 format.

        """

    subcomponents = Attribute("""
        A list of all subcomponents of this component,
        added using add_component()""")

    name = Attribute("""
        Name of this component (VEVENT, etc)
        """)

    def walk(name=None):
        """Recursively traverses component and subcomponents.

        name - optional, if given, only return components with that name

        Returns sequence of components.

        """

    def property_items(recursive=True):
        """Return properties as (name, value) tuples.

        Returns all properties in this comopnent and subcomponents as
        name, value tuples.

        """

class IEvent(IComponent):
    """A component which conforms to an iCalendar VEVENT.

    """

class ITodo(IComponent):
    """A component which conforms to an iCalendar VTODO.

    """

class IJournal(IComponent):
    """A component which conforms to an iCalendar VJOURNAL.

    """

class IFreeBusy(IComponent):
    """A component which conforms to an iCalendar VFREEBUSY.

    """

class ITimezone(IComponent):
    """A component which conforms to an iCalendar VTIMEZONE.

    """

class IAlarm(IComponent):
    """A component which conforms to an iCalendar VALARM.

    """

class ICalendar(IComponent):
    """A component which conforms to an iCalendar VCALENDAR.

    """

class IPropertyValue(Interface):
    """An iCalendar property value.
    iCalendar properties have strongly typed values.

    This invariance should always be true:

    assert x == vDataType.from_ical(vDataType(x).to_ical())

    """

    def to_ical():
        """Render property as string, as defined in iCalendar RFC 2445."""

    # this is a static method
    def from_ical(ical):
        """Parse property from iCalendar RFC 2445 text.
        Inverse of to_ical().

        """

class IBinary(IPropertyValue):
    """Binary property values are base 64 encoded."""

class IBoolean(IPropertyValue):
    """Boolean property.

    Also behaves like a python int.

    """

class ICalAddress(IPropertyValue):
    """Email address.

    Also behaves like a python str.

    """

class IDateTime(IPropertyValue):
    """Render and generates iCalendar datetime format.

    Important: if tzinfo is defined it renders itself as 'date with utc time'
    Meaning that it has a 'Z' appended, and is in absolute time.

    """

class IDate(IPropertyValue):
    """Render and generates iCalendar date format."""

class IDuration(IPropertyValue):
    """Render and generates timedelta in iCalendar DURATION format."""

class IFloat(IPropertyValue):
    """Render and generate floats in iCalendar format.

    Also behaves like a python float.

    """

class IInt(IPropertyValue):
    """Render and generate ints in iCalendar format.

    Also behaves like a python int.

    """

class IPeriod(IPropertyValue):
    """A precise period of time (datetime, datetime)."""

class IWeekDay(IPropertyValue):
    """Render and generate weekday abbreviation."""

class IFrequency(IPropertyValue):
    """Frequency."""

class IRecur(IPropertyValue):
    """Render and generate data based on recurrent event representation.

    This acts like a caseless dictionary.

    """

class IText(IPropertyValue):
    """Unicode text."""

class ITime(IPropertyValue):
    """Time."""

class IUri(IPropertyValue):
    """URI."""

class IGeo(IPropertyValue):
    """Geographical location."""

class IUTCOffset(IPropertyValue):
    """Offset from UTC."""

class IInline(IPropertyValue):
    """Inline list."""
<commit_msg>remove interfaces, as it is unused<commit_after><|endoftext|>


<commit_before>##############################################################################
# Copyright (c) 2013-2018, Lawrence Livermore National Security, LLC.
# Produced at the Lawrence Livermore National Laboratory.
#
# This file is part of Spack.
# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved.
# LLNL-CODE-647188
#
# For details, see https://github.com/spack/spack
# Please also see the NOTICE and LICENSE files for our notice and the LGPL.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License (as
# published by the Free Software Foundation) version 2.1, February 1999.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and
# conditions of the GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
##############################################################################

from spack import *
import re
import os
import glob

class Zoltan(Package):
    """The Zoltan library is a toolkit of parallel combinatorial algorithms
       for parallel, unstructured, and/or adaptive scientific
       applications.  Zoltan's largest component is a suite of dynamic
       load-balancing and partitioning algorithms that increase
       applications' parallel performance by reducing idle time.  Zoltan
       also has graph coloring and graph ordering algorithms, which are
       useful in task schedulers and parallel preconditioners.

    """

    homepage = "http://www.cs.sandia.gov/zoltan"
    url      = "http://www.cs.sandia.gov/~kddevin/Zoltan_Distributions/zoltan_distrib_v3.83.tar.gz"

    version('3.83', '1ff1bc93f91e12f2c533ddb01f2c095f')
    version('3.8', '9d8fba8a990896881b85351d4327c4a9')
    version('3.6', '9cce794f7241ecd8dbea36c3d7a880f9')
    version('3.3', '5eb8f00bda634b25ceefa0122bd18d65')

    variant('debug', default=False, description='Builds a debug version of the library.')
    variant('shared', default=True, description='Builds a shared version of the library.')

    variant('fortran', default=True, description='Enable Fortran support.')
    variant('mpi', default=True, description='Enable MPI support.')
    variant('parmetis', default=False, description='Enable ParMETIS support.')

    depends_on('mpi', when='+mpi')

    depends_on('parmetis@4:', when='+parmetis')
    depends_on('metis', when='+parmetis')

    conflicts('+parmetis', when='~mpi')

    def install(self, spec, prefix):
        # FIXME: The older Zoltan versions fail to compile the F90 MPI wrappers
        # because of some complicated generic type problem.
        if spec.satisfies('@:3.6+fortran+mpi'):
            raise RuntimeError(('Cannot build Zoltan v{0} with +fortran and '
                                '+mpi; please disable one of these features '
                                'or upgrade versions.').format(self.version))

        config_args = [
            self.get_config_flag('f90interface', 'fortran'),
            self.get_config_flag('mpi', 'mpi'),
        ]
        config_cflags = [
            '-O0' if '+debug' in spec else '-O3',
            '-g' if '+debug' in spec else '',
        ]

        if '+shared' in spec:
            config_args.append('RANLIB=echo')
            config_args.append('--with-ar=$(CXX) -shared $(LDFLAGS) -o')
            config_cflags.append(self.compiler.pic_flag)
            if spec.satisfies('%gcc'):
                config_args.append('--with-libs={0}'.format('-lgfortran'))

        if '+parmetis' in spec:
            config_args.append('--with-parmetis')
            config_args.append('--with-parmetis-libdir={0}'
                               .format(spec['parmetis'].prefix.lib))
            config_args.append('--with-parmetis-incdir={0}'
                               .format(spec['parmetis'].prefix.include))
            config_args.append('--with-incdirs=-I{0}'
                               .format(spec['metis'].prefix.include))
            config_args.append('--with-ldflags=-L{0}'
                               .format(spec['metis'].prefix.lib))

        if '+mpi' in spec:
            config_args.append('CC={0}'.format(spec['mpi'].mpicc))
            config_args.append('CXX={0}'.format(spec['mpi'].mpicxx))
            config_args.append('FC={0}'.format(spec['mpi'].mpifc))

            config_args.append('--with-mpi={0}'.format(spec['mpi'].prefix))

            mpi_libs = self.get_mpi_libs()

            # NOTE: Some external mpi installations may have empty lib
            # directory (e.g. bg-q). In this case we need to explicitly
            # pass empty library name.
            if mpi_libs:
                mpi_libs = ' -l'.join(mpi_libs)
                config_args.append('--with-mpi-libs=-l{0}'.format(mpi_libs))
            else:
                config_args.append('--with-mpi-libs= ')

        # NOTE: Early versions of Zoltan come packaged with a few embedded
        # library packages (e.g. ParMETIS, Scotch), which messes with Spack's
        # ability to descend directly into the package's source directory.
        source_directory = self.stage.source_path
        if spec.satisfies('@:3.6'):
            zoltan_directory = 'Zoltan_v{0}'.format(self.version)
            source_directory = join_path(source_directory, zoltan_directory)

        build_directory = join_path(source_directory, 'build')
        with working_dir(build_directory, create=True):
            config = Executable(join_path(source_directory, 'configure'))
            config(
                '--prefix={0}'.format(prefix),
                '--with-cflags={0}'.format(' '.join(config_cflags)),
                '--with-cxxflags={0}'.format(' '.join(config_cflags)),
                '--with-fcflags={0}'.format(' '.join(config_cflags)),
                *config_args
            )

            # NOTE: Earlier versions of Zoltan cannot be built in parallel
            # because they contain nested Makefile dependency bugs.
            make(parallel=not spec.satisfies('@:3.6+fortran'))
            make('install')

        # NOTE: Unfortunately, Zoltan doesn't provide any configuration
        # options for the extension of the output library files, so this
        # script must change these extensions as a post-processing step.
        if '+shared' in spec:
            for lib_path in glob.glob(join_path(prefix, 'lib', '*.a')):
                lib_static_name = os.path.basename(lib_path)
                lib_shared_name = re.sub(r'\.a$', '.{0}'.format(dso_suffix),
                                         lib_static_name)
                move(lib_path, join_path(prefix, 'lib', lib_shared_name))

    def get_config_flag(self, flag_name, flag_variant):
        flag_pre = 'en' if '+{0}'.format(flag_variant) in self.spec else 'dis'
        return '--{0}able-{1}'.format(flag_pre, flag_name)

    # NOTE: Zoltan assumes that it's linking against an MPI library that can
    # be found with '-lmpi,' which isn't the case for many MPI packages.  This
    # function finds the names of the actual libraries for Zoltan's MPI dep.
    def get_mpi_libs(self):
        mpi_libs = set()

        for lib_path in glob.glob(join_path(self.spec['mpi'].prefix.lib, '*')):
            mpi_lib_match = re.match(
                r'^(lib)((\w*)mpi(\w*))\.((a)|({0}))$'.format(dso_suffix),
                os.path.basename(lib_path))
            if mpi_lib_match:
                mpi_libs.add(mpi_lib_match.group(2))

        return list(mpi_libs)
<commit_msg>zoltan: fix building with Intel compilers (#8985)<commit_after>##############################################################################
# Copyright (c) 2013-2018, Lawrence Livermore National Security, LLC.
# Produced at the Lawrence Livermore National Laboratory.
#
# This file is part of Spack.
# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved.
# LLNL-CODE-647188
#
# For details, see https://github.com/spack/spack
# Please also see the NOTICE and LICENSE files for our notice and the LGPL.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License (as
# published by the Free Software Foundation) version 2.1, February 1999.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and
# conditions of the GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
##############################################################################

from spack import *
import re
import os
import glob

class Zoltan(Package):
    """The Zoltan library is a toolkit of parallel combinatorial algorithms
       for parallel, unstructured, and/or adaptive scientific
       applications.  Zoltan's largest component is a suite of dynamic
       load-balancing and partitioning algorithms that increase
       applications' parallel performance by reducing idle time.  Zoltan
       also has graph coloring and graph ordering algorithms, which are
       useful in task schedulers and parallel preconditioners.

    """

    homepage = "http://www.cs.sandia.gov/zoltan"
    url      = "http://www.cs.sandia.gov/~kddevin/Zoltan_Distributions/zoltan_distrib_v3.83.tar.gz"

    version('3.83', '1ff1bc93f91e12f2c533ddb01f2c095f')
    version('3.8', '9d8fba8a990896881b85351d4327c4a9')
    version('3.6', '9cce794f7241ecd8dbea36c3d7a880f9')
    version('3.3', '5eb8f00bda634b25ceefa0122bd18d65')

    variant('debug', default=False, description='Builds a debug version of the library.')
    variant('shared', default=True, description='Builds a shared version of the library.')

    variant('fortran', default=True, description='Enable Fortran support.')
    variant('mpi', default=True, description='Enable MPI support.')
    variant('parmetis', default=False, description='Enable ParMETIS support.')

    depends_on('mpi', when='+mpi')

    depends_on('parmetis@4:', when='+parmetis')
    depends_on('metis', when='+parmetis')

    conflicts('+parmetis', when='~mpi')

    def install(self, spec, prefix):
        # FIXME: The older Zoltan versions fail to compile the F90 MPI wrappers
        # because of some complicated generic type problem.
        if spec.satisfies('@:3.6+fortran+mpi'):
            raise RuntimeError(('Cannot build Zoltan v{0} with +fortran and '
                                '+mpi; please disable one of these features '
                                'or upgrade versions.').format(self.version))

        config_args = [
            self.get_config_flag('f90interface', 'fortran'),
            self.get_config_flag('mpi', 'mpi'),
        ]
        config_cflags = [
            '-O0' if '+debug' in spec else '-O3',
            '-g' if '+debug' in spec else '',
        ]

        if '+shared' in spec:
            config_args.append('RANLIB=echo')
            config_args.append('--with-ar=$(CXX) -shared $(LDFLAGS) -o')
            config_cflags.append(self.compiler.pic_flag)
            if spec.satisfies('%gcc'):
                config_args.append('--with-libs=-lgfortran')
            if spec.satisfies('%intel'):
                config_args.append('--with-libs=-lifcore')

        if '+parmetis' in spec:
            config_args.append('--with-parmetis')
            config_args.append('--with-parmetis-libdir={0}'
                               .format(spec['parmetis'].prefix.lib))
            config_args.append('--with-parmetis-incdir={0}'
                               .format(spec['parmetis'].prefix.include))
            config_args.append('--with-incdirs=-I{0}'
                               .format(spec['metis'].prefix.include))
            config_args.append('--with-ldflags=-L{0}'
                               .format(spec['metis'].prefix.lib))

        if '+mpi' in spec:
            config_args.append('CC={0}'.format(spec['mpi'].mpicc))
            config_args.append('CXX={0}'.format(spec['mpi'].mpicxx))
            config_args.append('FC={0}'.format(spec['mpi'].mpifc))

            config_args.append('--with-mpi={0}'.format(spec['mpi'].prefix))

            mpi_libs = self.get_mpi_libs()

            # NOTE: Some external mpi installations may have empty lib
            # directory (e.g. bg-q). In this case we need to explicitly
            # pass empty library name.
            if mpi_libs:
                mpi_libs = ' -l'.join(mpi_libs)
                config_args.append('--with-mpi-libs=-l{0}'.format(mpi_libs))
            else:
                config_args.append('--with-mpi-libs= ')

        # NOTE: Early versions of Zoltan come packaged with a few embedded
        # library packages (e.g. ParMETIS, Scotch), which messes with Spack's
        # ability to descend directly into the package's source directory.
        source_directory = self.stage.source_path
        if spec.satisfies('@:3.6'):
            zoltan_directory = 'Zoltan_v{0}'.format(self.version)
            source_directory = join_path(source_directory, zoltan_directory)

        build_directory = join_path(source_directory, 'build')
        with working_dir(build_directory, create=True):
            config = Executable(join_path(source_directory, 'configure'))
            config(
                '--prefix={0}'.format(prefix),
                '--with-cflags={0}'.format(' '.join(config_cflags)),
                '--with-cxxflags={0}'.format(' '.join(config_cflags)),
                '--with-fcflags={0}'.format(' '.join(config_cflags)),
                *config_args
            )

            # NOTE: Earlier versions of Zoltan cannot be built in parallel
            # because they contain nested Makefile dependency bugs.
            make(parallel=not spec.satisfies('@:3.6+fortran'))
            make('install')

        # NOTE: Unfortunately, Zoltan doesn't provide any configuration
        # options for the extension of the output library files, so this
        # script must change these extensions as a post-processing step.
        if '+shared' in spec:
            for lib_path in glob.glob(join_path(prefix, 'lib', '*.a')):
                lib_static_name = os.path.basename(lib_path)
                lib_shared_name = re.sub(r'\.a$', '.{0}'.format(dso_suffix),
                                         lib_static_name)
                move(lib_path, join_path(prefix, 'lib', lib_shared_name))

    def get_config_flag(self, flag_name, flag_variant):
        flag_pre = 'en' if '+{0}'.format(flag_variant) in self.spec else 'dis'
        return '--{0}able-{1}'.format(flag_pre, flag_name)

    # NOTE: Zoltan assumes that it's linking against an MPI library that can
    # be found with '-lmpi,' which isn't the case for many MPI packages.  This
    # function finds the names of the actual libraries for Zoltan's MPI dep.
    def get_mpi_libs(self):
        mpi_libs = set()

        for lib_path in glob.glob(join_path(self.spec['mpi'].prefix.lib, '*')):
            mpi_lib_match = re.match(
                r'^(lib)((\w*)mpi(\w*))\.((a)|({0}))$'.format(dso_suffix),
                os.path.basename(lib_path))
            if mpi_lib_match:
                mpi_libs.add(mpi_lib_match.group(2))

        return list(mpi_libs)
<|endoftext|>


<commit_before>#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#
# Copyright (c) 2013 Frederik Elwert <frederik.elwert@web.de>
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.

"""
A POS tagset based on ISOcat DC-1345.

"""

import os

from tcflib.tagsets.base import POSTagSetBase, POSTagBase, register_tagset

class POSTag(POSTagBase):
    # See http://www.sfs.uni-tuebingen.de/Elwis/stts/Wortlisten/WortFormen.html
    CLOSED = [
        'http://www.isocat.org/datcat/DC-1372',  # punctuation
        'http://www.isocat.org/datcat/DC-1231',  # adposition
        'http://www.isocat.org/datcat/DC-1892',  # article
        'http://www.isocat.org/datcat/DC-1342',  # particle
        'http://www.isocat.org/datcat/DC-1260',  # conjunction
        'http://www.isocat.org/datcat/DC-2998',  # pronominal adverb
        'http://www.isocat.org/datcat/DC-1370',  # pronoun
        'http://www.isocat.org/datcat/DC-1263',  # copula
        'http://www.isocat.org/datcat/DC-1329',  # modal
        'http://www.isocat.org/datcat/DC-1891',  # unclassified residual
        'http://www.isocat.org/datcat/DC-1908',  # unspecified
        ]

class POSTagSet(POSTagSetBase):
    __file__ = os.path.join(os.path.dirname(__file__),
                            'data', 'dc-1345.dcif')
    name = 'DC-1345'
    pid = 'http://www.isocat.org/datcat/DC-1345'
    tag_class = POSTag

register_tagset(POSTagSet)
<commit_msg>Use more generic category 'determiner' instead of 'article' for closed tags.<commit_after>#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#
# Copyright (c) 2013 Frederik Elwert <frederik.elwert@web.de>
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.

"""
A POS tagset based on ISOcat DC-1345.

"""

import os

from tcflib.tagsets.base import POSTagSetBase, POSTagBase, register_tagset

class POSTag(POSTagBase):
    # See http://www.sfs.uni-tuebingen.de/Elwis/stts/Wortlisten/WortFormen.html
    CLOSED = [
        'http://www.isocat.org/datcat/DC-1372',  # punctuation
        'http://www.isocat.org/datcat/DC-1231',  # adposition
        #'http://www.isocat.org/datcat/DC-1892',  # article
        'http://www.isocat.org/datcat/DC-1272', # determiner; includes article
        'http://www.isocat.org/datcat/DC-1342',  # particle
        'http://www.isocat.org/datcat/DC-1260',  # conjunction
        'http://www.isocat.org/datcat/DC-2998',  # pronominal adverb
        'http://www.isocat.org/datcat/DC-1370',  # pronoun
        'http://www.isocat.org/datcat/DC-1263',  # copula
        'http://www.isocat.org/datcat/DC-1329',  # modal
        'http://www.isocat.org/datcat/DC-1891',  # unclassified residual
        'http://www.isocat.org/datcat/DC-1908',  # unspecified
        ]

class POSTagSet(POSTagSetBase):
    __file__ = os.path.join(os.path.dirname(__file__),
                            'data', 'dc-1345.dcif')
    name = 'DC-1345'
    pid = 'http://www.isocat.org/datcat/DC-1345'
    tag_class = POSTag

register_tagset(POSTagSet)
<|endoftext|>


<commit_before>#!/usr/bin/env python
# =========================================================================
#
#  Copyright NumFOCUS
#
#  Licensed under the Apache License, Version 2.0 (the "License");
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0.txt
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.
#
# =========================================================================

import SimpleITK as sitk
import sys
import os

if len(sys.argv) < 2:
    print("Usage: N4BiasFieldCorrection inputImage " +
          "outputImage [shrinkFactor] [maskImage] [numberOfIterations] " +
          "[numberOfFittingLevels]")
    sys.exit(1)

inputImage = sitk.ReadImage(sys.argv[1], sitk.sitkFloat32)
image = inputImage

if len(sys.argv) > 4:
    maskImage = sitk.ReadImage(sys.argv[4], sitk.sitkUint8)
else:
    maskImage = sitk.OtsuThreshold(inputImage, 0, 1, 200)

if len(sys.argv) > 3:
    image = sitk.Shrink(inputImage,
                             [int(sys.argv[3])] * inputImage.GetDimension())
    maskImage = sitk.Shrink(maskImage,
                            [int(sys.argv[3])] * inputImage.GetDimension())

corrector = sitk.N4BiasFieldCorrectionImageFilter()

numberFittingLevels = 4

if len(sys.argv) > 6:
    numberFittingLevels = int(sys.argv[6])

if len(sys.argv) > 5:
    corrector.SetMaximumNumberOfIterations([int(sys.argv[5])]
                                           * numberFittingLevels)

corrected_image = corrector.Execute(image, maskImage)

log_bias_field = corrector.GetLogBiasFieldAsImage(inputImage)

corrected_image_full_resolution = inputImage / sitk.Exp( log_bias_field )

sitk.WriteImage(corrected_image, sys.argv[2])

if ("SITK_NOSHOW" not in os.environ):
    sitk.Show(corrected_image, "N4 Corrected")
<commit_msg>bugfix (typo in sitkUint8)<commit_after>#!/usr/bin/env python
# =========================================================================
#
#  Copyright NumFOCUS
#
#  Licensed under the Apache License, Version 2.0 (the "License");
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0.txt
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.
#
# =========================================================================

import SimpleITK as sitk
import sys
import os

if len(sys.argv) < 2:
    print("Usage: N4BiasFieldCorrection inputImage " +
          "outputImage [shrinkFactor] [maskImage] [numberOfIterations] " +
          "[numberOfFittingLevels]")
    sys.exit(1)

inputImage = sitk.ReadImage(sys.argv[1], sitk.sitkFloat32)
image = inputImage

if len(sys.argv) > 4:
    maskImage = sitk.ReadImage(sys.argv[4], sitk.sitkUInt8)
else:
    maskImage = sitk.OtsuThreshold(inputImage, 0, 1, 200)

if len(sys.argv) > 3:
    image = sitk.Shrink(inputImage,
                             [int(sys.argv[3])] * inputImage.GetDimension())
    maskImage = sitk.Shrink(maskImage,
                            [int(sys.argv[3])] * inputImage.GetDimension())

corrector = sitk.N4BiasFieldCorrectionImageFilter()

numberFittingLevels = 4

if len(sys.argv) > 6:
    numberFittingLevels = int(sys.argv[6])

if len(sys.argv) > 5:
    corrector.SetMaximumNumberOfIterations([int(sys.argv[5])]
                                           * numberFittingLevels)

corrected_image = corrector.Execute(image, maskImage)

log_bias_field = corrector.GetLogBiasFieldAsImage(inputImage)

corrected_image_full_resolution = inputImage / sitk.Exp( log_bias_field )

sitk.WriteImage(corrected_image, sys.argv[2])

if ("SITK_NOSHOW" not in os.environ):
    sitk.Show(corrected_image, "N4 Corrected")
<|endoftext|>


<commit_before>import itertools
import typing
import random
import string
from typing import TypeVar, Callable, Generic, Iterable, Any
from functools import reduce

from toolz.itertoolz import cons

from amino import maybe, boolean
from amino.logging import log
from amino.tc.base import ImplicitsMeta, Implicits
from amino.func import curried, I
from amino.util.string import safe_string

A = TypeVar('A', covariant=True)
B = TypeVar('B')

def flatten(l: Iterable[Iterable[A]]) -> Iterable[A]:
    return list(itertools.chain.from_iterable(l))  # type: ignore

class ListMeta(ImplicitsMeta):

    def __instancecheck__(self, instance):
        if type(instance) is list:
            return False
        else:
            return super().__instancecheck__(instance)

    def __subclasscheck__(self, subclass):
        if subclass is list:
            return False
        return super().__subclasscheck__(subclass)

class List(typing.List[A], Generic[A], Implicits, implicits=True,
           metaclass=ListMeta):

    def __init__(self, *elements):
        typing.List.__init__(self, elements)

    def __getitem__(self, arg):
        s = super().__getitem__(arg)
        return List.wrap(s) if isinstance(arg, slice) else s

    @staticmethod
    def wrap(l: Iterable[B]) -> 'List[B]':
        return List(*list(l))

    @staticmethod
    def range(*a):
        return List.wrap(range(*a))

    @staticmethod
    def random_string(num: int=10):
        chars = string.ascii_letters + string.digits
        return ''.join(random.choice(chars) for i in range(num))

    @staticmethod
    def gen(num: int, f: Callable[[], A]):
        return List.range(num) // (lambda a: f())

    @staticmethod
    def lines(data: str):
        return List.wrap(data.splitlines())

    def lift(self, index: int) -> 'maybe.Maybe[A]':
        return (
            (maybe.Just(self[index]) if len(self) > index else maybe.Empty())
            if index >= 0 else
            (maybe.Just(self[index]) if len(self) >= -index else maybe.Empty())
        )

    def lift_all(self, index, *indexes):
        from amino.anon import _
        def folder(z, n):
            return n.ap(z / _.cat)
        els = List.wrap(indexes) / self.lift
        init = self.lift(index) / List
        return els.fold_left(init)(folder)

    def smap(self, f: Callable[..., B]) -> 'List[B]':
        return List.wrap(list(itertools.starmap(f, self)))

    def flat_smap(self, f: Callable[..., 'Iterable[B]']) -> 'List[B]':
        return List.wrap(flatten(list(itertools.starmap(f, self))))

    def foreach(self, f: Callable[[A], B]) -> None:
        for el in self:
            f(el)

    def forall(self, f: Callable[[A], bool]) -> boolean.Boolean:
        return boolean.Boolean(all(f(el) for el in self))

    def contains(self, value):
        return value in self

    def exists(self, f: Callable[[A], bool]) -> bool:
        return self.find(f).is_just

    @property
    def is_empty(self):
        return boolean.Boolean(self.length == 0)

    empty = is_empty

    @property
    def length(self):
        return len(self)

    @property
    def head(self):
        return self.lift(0)

    @property
    def last(self):
        return self.lift(-1)

    @property
    def init(self):
        return maybe.Empty() if self.empty else maybe.Just(self[:-1])

    @property
    def tail(self):
        return maybe.Empty() if self.empty else maybe.Just(self[1:])

    @property
    def detach_head(self):
        return self.head.product(self.tail)

    @property
    def detach_last(self):
        return self.last.product(self.init)

    @property
    def distinct(self) -> 'List[A]':
        return self.distinct_by(I)

    def distinct_by(self, f: Callable[[A], bool]) -> 'List[A]':
        seen = set()  # type: set
        def pred(a):
            v = f(a)
            return v in seen or seen.add(v)
        return List.wrap(x for x in self if not pred(x))

    def add(self, other: typing.List[A]) -> 'List[A]':
        return List.wrap(typing.List.__add__(self, other))

    __add__ = add

    def without(self, el) -> 'List[A]':
        from amino.anon import _
        return self.filter(_ != el)

    __sub__ = without

    def split(self, f: Callable[[A], bool]):
        def splitter(z, e):
            l, r = z
            return (l + (e,), r) if f(e) else (l, r + (e,))
        l, r = reduce(splitter, self, ((), (),))
        return List.wrap(l), List.wrap(r)

    def split_type(self, tpe: type):
        return self.split(lambda a: isinstance(a, tpe))

    def debug(self, prefix=None):
        prefix = '' if prefix is None else prefix + ' '
        log.debug(prefix + str(self))
        return self

    def index_of(self, target: Any):
        from amino.anon import _
        return self.index_where(_ == target)

    @property
    def reversed(self):
        return List.wrap(reversed(self))

    def mk_string(self, sep=''):
        return sep.join(self / str)

    @property
    def join_lines(self):
        return self.mk_string('\n')

    @property
    def join_comma(self):
        return self.mk_string(', ')

    @property
    def join_tokens(self):
        return self.mk_string(' ')

    def cons(self, item):
        return List.wrap(cons(item, self))

    def cat(self, item):
        return self + List(item)

    def cat_m(self, item: maybe.Maybe):
        return item / self.cat | self

    @property
    def transpose(self):
        return List.wrap(map(List.wrap, zip(*self)))

    def drop(self, n: int):
        return self[n:]

    def take(self, n: int):
        return self[:n]

    def drop_while(self, pred: Callable[[A], bool]):
        index = self.index_where(lambda a: not pred(a))
        return index / (lambda a: self[a:]) | self

    def remove_all(self, els: 'List[A]') -> 'List[A]':
        return self.filter_not(els.contains)

    def __repr__(self):
        return 'List({})'.format(', '.join(map(repr, self)))

    def __str__(self):
        return '[{}]'.format(', '.join(map(safe_string, self)))

    def __hash__(self):
        return hash(tuple(self))

    def sort_by(self, f: Callable[[A], bool], reverse=False):
        return List.wrap(sorted(self, key=f, reverse=reverse))

    def sort(self, reverse=False):
        return self.sort_by(I, reverse)

    def replace_item(self, a, b) -> 'List[A]':
        return self.map(lambda c: b if c == a else c)

    @curried
    def replace_where(self, a, pred: Callable):
        return self.map(lambda b: a if pred(b) else b)

    def __mul__(self, other):
        return List.wrap(super().__mul__(other))

__all__ = ('List',)
<commit_msg>`List.random_alpha`<commit_after>import itertools
import typing
import random
import string
from typing import TypeVar, Callable, Generic, Iterable, Any
from functools import reduce

from toolz.itertoolz import cons

from amino import maybe, boolean
from amino.logging import log
from amino.tc.base import ImplicitsMeta, Implicits
from amino.func import curried, I
from amino.util.string import safe_string

A = TypeVar('A', covariant=True)
B = TypeVar('B')

def flatten(l: Iterable[Iterable[A]]) -> Iterable[A]:
    return list(itertools.chain.from_iterable(l))  # type: ignore

class ListMeta(ImplicitsMeta):

    def __instancecheck__(self, instance):
        if type(instance) is list:
            return False
        else:
            return super().__instancecheck__(instance)

    def __subclasscheck__(self, subclass):
        if subclass is list:
            return False
        return super().__subclasscheck__(subclass)

def _rand_str(chars, num: int=10):
    return ''.join(random.choice(chars) for i in range(num))

class List(typing.List[A], Generic[A], Implicits, implicits=True,
           metaclass=ListMeta):

    def __init__(self, *elements):
        typing.List.__init__(self, elements)

    def __getitem__(self, arg):
        s = super().__getitem__(arg)
        return List.wrap(s) if isinstance(arg, slice) else s

    @staticmethod
    def wrap(l: Iterable[B]) -> 'List[B]':
        return List(*list(l))

    @staticmethod
    def range(*a):
        return List.wrap(range(*a))

    @staticmethod
    def random_string(num: int=10):
        chars = string.ascii_letters + string.digits
        return _rand_str(chars, num)

    @staticmethod
    def random_alpha(num: int=10):
        chars = string.ascii_letters
        return _rand_str(chars, num)

    @staticmethod
    def gen(num: int, f: Callable[[], A]):
        return List.range(num) // (lambda a: f())

    @staticmethod
    def lines(data: str):
        return List.wrap(data.splitlines())

    def lift(self, index: int) -> 'maybe.Maybe[A]':
        return (
            (maybe.Just(self[index]) if len(self) > index else maybe.Empty())
            if index >= 0 else
            (maybe.Just(self[index]) if len(self) >= -index else maybe.Empty())
        )

    def lift_all(self, index, *indexes):
        from amino.anon import _
        def folder(z, n):
            return n.ap(z / _.cat)
        els = List.wrap(indexes) / self.lift
        init = self.lift(index) / List
        return els.fold_left(init)(folder)

    def smap(self, f: Callable[..., B]) -> 'List[B]':
        return List.wrap(list(itertools.starmap(f, self)))

    def flat_smap(self, f: Callable[..., 'Iterable[B]']) -> 'List[B]':
        return List.wrap(flatten(list(itertools.starmap(f, self))))

    def foreach(self, f: Callable[[A], B]) -> None:
        for el in self:
            f(el)

    def forall(self, f: Callable[[A], bool]) -> boolean.Boolean:
        return boolean.Boolean(all(f(el) for el in self))

    def contains(self, value):
        return value in self

    def exists(self, f: Callable[[A], bool]) -> bool:
        return self.find(f).is_just

    @property
    def is_empty(self):
        return boolean.Boolean(self.length == 0)

    empty = is_empty

    @property
    def length(self):
        return len(self)

    @property
    def head(self):
        return self.lift(0)

    @property
    def last(self):
        return self.lift(-1)

    @property
    def init(self):
        return maybe.Empty() if self.empty else maybe.Just(self[:-1])

    @property
    def tail(self):
        return maybe.Empty() if self.empty else maybe.Just(self[1:])

    @property
    def detach_head(self):
        return self.head.product(self.tail)

    @property
    def detach_last(self):
        return self.last.product(self.init)

    @property
    def distinct(self) -> 'List[A]':
        return self.distinct_by(I)

    def distinct_by(self, f: Callable[[A], bool]) -> 'List[A]':
        seen = set()  # type: set
        def pred(a):
            v = f(a)
            return v in seen or seen.add(v)
        return List.wrap(x for x in self if not pred(x))

    def add(self, other: typing.List[A]) -> 'List[A]':
        return List.wrap(typing.List.__add__(self, other))

    __add__ = add

    def without(self, el) -> 'List[A]':
        from amino.anon import _
        return self.filter(_ != el)

    __sub__ = without

    def split(self, f: Callable[[A], bool]):
        def splitter(z, e):
            l, r = z
            return (l + (e,), r) if f(e) else (l, r + (e,))
        l, r = reduce(splitter, self, ((), (),))
        return List.wrap(l), List.wrap(r)

    def split_type(self, tpe: type):
        return self.split(lambda a: isinstance(a, tpe))

    def debug(self, prefix=None):
        prefix = '' if prefix is None else prefix + ' '
        log.debug(prefix + str(self))
        return self

    def index_of(self, target: Any):
        from amino.anon import _
        return self.index_where(_ == target)

    @property
    def reversed(self):
        return List.wrap(reversed(self))

    def mk_string(self, sep=''):
        return sep.join(self / str)

    @property
    def join_lines(self):
        return self.mk_string('\n')

    @property
    def join_comma(self):
        return self.mk_string(', ')

    @property
    def join_tokens(self):
        return self.mk_string(' ')

    def cons(self, item):
        return List.wrap(cons(item, self))

    def cat(self, item):
        return self + List(item)

    def cat_m(self, item: maybe.Maybe):
        return item / self.cat | self

    @property
    def transpose(self):
        return List.wrap(map(List.wrap, zip(*self)))

    def drop(self, n: int):
        return self[n:]

    def take(self, n: int):
        return self[:n]

    def drop_while(self, pred: Callable[[A], bool]):
        index = self.index_where(lambda a: not pred(a))
        return index / (lambda a: self[a:]) | self

    def remove_all(self, els: 'List[A]') -> 'List[A]':
        return self.filter_not(els.contains)

    def __repr__(self):
        return 'List({})'.format(', '.join(map(repr, self)))

    def __str__(self):
        return '[{}]'.format(', '.join(map(safe_string, self)))

    def __hash__(self):
        return hash(tuple(self))

    def sort_by(self, f: Callable[[A], bool], reverse=False):
        return List.wrap(sorted(self, key=f, reverse=reverse))

    def sort(self, reverse=False):
        return self.sort_by(I, reverse)

    def replace_item(self, a, b) -> 'List[A]':
        return self.map(lambda c: b if c == a else c)

    @curried
    def replace_where(self, a, pred: Callable):
        return self.map(lambda b: a if pred(b) else b)

    def __mul__(self, other):
        return List.wrap(super().__mul__(other))

__all__ = ('List',)
<|endoftext|>