<commit_before>#!/usr/bin/env python

import argparse, datetime, getpass, os, sys, shutil, traceback
from fabric.api import run, cd, local, get, settings, lcd, put
from fabric_ssh_config import getSSHInfoForHost
from fabric.context_managers import shell_env
from fabric.utils import abort

#Login as user test, but build in a directory by real username
username = 'test'
builddir = "/tmp/" + getpass.getuser() + "Kits/buildtemp"
version = "UNKNOWN"
nativelibdir = "/nativelibs/obj"  #  ~test/libs/... usually
defaultlicensedays = 70 #default trial license length

################################################
# CHECKOUT CODE INTO A TEMP DIR
################################################

def checkoutCode(voltdbGit, proGit, rbmqExportGit, gitloc):
    global buildir
    # clean out the existing dir
    run("rm -rf " + builddir)
    # make the build dir again
    run("mkdir -p " + builddir)
    # change to it
    with cd(builddir):
        # do the checkouts, collect checkout errors on both community &
        # pro repos so user gets status on both checkouts
        message = ""
        run("git clone -q %s/voltdb.git" % gitloc)
        result = run("cd voltdb; git checkout %s" % voltdbGit, warn_only=True)
        if result.failed:
            message = "VoltDB checkout failed. Missing branch %s." % rbmqExportGit

        run("git clone -q %s/pro.git" % gitloc)
        result = run("cd pro; git checkout %s" % proGit, warn_only=True)
        if result.failed:
            message += "\nPro checkout failed. Missing branch %s." % rbmqExportGit

        #rabbitmq isn't mirrored internally, so don't use gitloc
        run("git clone -q git@github.com:VoltDB/export-rabbitmq.git")
        result = run("cd export-rabbitmq; git checkout %s" % rbmqExportGit, warn_only=True)
        # Probably ok to use master for export-rabbitmq.
        if result.failed:
            print "\nExport-rabbitmg branch %s checkout failed. Defaulting to master." % rbmqExportGit

        if len(message) > 0:
            abort(message)

        return run("cat voltdb/version.txt").strip()

################################################
# MAKE A RELEASE DIR
################################################

def makeReleaseDir(releaseDir):
    # handle the case where a release dir exists for this version
    if os.path.exists(releaseDir):
        shutil.rmtree(releaseDir)
    # create a release dir
    os.makedirs(releaseDir)
    print "Created dir: " + releaseDir

################################################
# BUILD THE COMMUNITY VERSION
################################################

def buildCommunity():
    if build_mac:
        packageMacLib="true"
    else:
        packageMacLib="false"
    with cd(builddir + "/voltdb"):
        run("pwd")
        run("git status")
        run("git describe --dirty")
        run("ant -Djmemcheck=NO_MEMCHECK -Dkitbuild=%s %s clean default dist" % (packageMacLib,  build_args))

################################################
# BUILD THE ENTERPRISE VERSION
################################################

def buildEnterprise():
    if build_mac:
        packageMacLib="true"
    else:
        packageMacLib="false"
    with cd(builddir + "/pro"):
        run("pwd")
        run("git status")
        run("git describe --dirty")
        run("VOLTCORE=../voltdb ant -f mmt.xml -Djmemcheck=NO_MEMCHECK -Dallowreplication=true -DallowDrActiveActive=true -Dlicensedays=%d -Dkitbuild=%s %s clean dist.pro" % (defaultlicensedays, packageMacLib, build_args))

################################################
# BUILD THE PRO VERSION
################################################

#
def packagePro(version):
    print "Making license"
    makeTrialLicense(days=defaultlicensedays, dr_and_xdcr=False, nodes=3)
    print "Repacking pro kit"
    with cd(builddir + "/pro/obj/pro"):
        run("mkdir pro_kit_staging")
    with cd(builddir + "/pro/obj/pro/pro_kit_staging"):
        run("tar xf ../voltdb-ent-%s.tar.gz" % version)
        run("mv voltdb-ent-%s voltdb-pro-%s" % (version, version))
        run("cp %s/pro/trial_*.xml voltdb-pro-%s/voltdb/license.xml" % (builddir, version))
        run("tar cvfz ../voltdb-pro-%s.tar.gz voltdb-pro-%s" % (version, version))

################################################
# BUILD THE RABBITMQ EXPORT CONNECTOR
################################################
#Build rabbit MQ Exporter
def buildRabbitMQExport(version, dist_type):
    # Paths to the final kit for unpacking/repacking with rmq export
    paths = {
        'community': builddir + "/voltdb/obj/release",
        'ent' : builddir + "/pro/obj/pro/"
        }
    # Untar
    with cd(paths[dist_type]):
        run ("pwd")
        run ("mkdir -p restage")
        run ("tar xf voltdb-%s-%s.tar.gz -C restage" % (dist_type, version))
        run ("rm -f voltdb-%s-%s.tar.gz" % (dist_type, version))

    # Build RabbitMQ export jar and put it into the untarred kit
    with cd(builddir + "/export-rabbitmq"):
        run("pwd")
        run("git status")
        run("git describe --dirty", warn_only=True)
        run("VOLTDIST=%s/restage/voltdb-%s-%s ant" % (paths[dist_type], dist_type, version))

    # Retar
    with cd(paths[dist_type]):
        run("pwd")
        run("tar -C restage -czf voltdb-%s-%s.tar.gz voltdb-%s-%s" % (dist_type, version, dist_type, version))
        run ("rm -Rf restage")

################################################
# MAKE AN ENTERPRISE TRIAL LICENSE
################################################

# Must be called after buildEnterprise has been done
def makeTrialLicense(days=30, dr_and_xdcr="true", nodes=12):
    with cd(builddir + "/pro/tools"):
        run("./make_trial_licenses.pl -t %d -H %d -W %s" % (days, nodes, dr_and_xdcr ))

################################################
# MAKE A SHA256 checksum
################################################

def makeSHA256SUM(version, type):
    with cd(builddir + "/pro/obj/pro"):
        kitname="voltdb-" +  type + "-" + version
        run("sha256sum -b %s.tar.gz > %s.SHA256SUM" % (kitname, kitname))

################################################
# MAKE AN JAR FILES NEEDED TO PUSH TO MAVEN
################################################

def makeMavenJars():
    with cd(builddir + "/voltdb"):
        run("VOLTCORE=../voltdb ant -f build-client.xml maven-jars")

################################################
# COPY FILES
################################################

def copyFilesToReleaseDir(releaseDir, version, type=None):
    print "Copying files to releaseDir"
    if type:
        typeString="-" + type
    else:
        typeString=""
    get("%s/pro/obj/pro/voltdb%s-%s.tar.gz" % (builddir, typeString, version),
        "%s/voltdb%s-%s.tar.gz" % (releaseDir, typeString, version))
    get("%s/pro/obj/pro/voltdb%s-%s.SHA256SUM" % (builddir, typeString, version),
        "%s/voltdb%s-%s.SHA256SUM" % (releaseDir, typeString, version))

def copyCommunityFilesToReleaseDir(releaseDir, version, operatingsys):
    get("%s/voltdb/obj/release/voltdb-community-%s.tar.gz" % (builddir, version),
        "%s/voltdb-community-%s.tar.gz" % (releaseDir, version))

    # add stripped symbols
    if operatingsys == "LINUX":
        os.makedirs(releaseDir + "/other")
        get("%s/voltdb/obj/release/voltdb-%s.sym" % (builddir, version),
            "%s/other/%s-voltdb-voltkv-%s.sym" % (releaseDir, operatingsys, version))

def copyTrialLicenseToReleaseDir(releaseDir):
    get("%s/pro/trial_*.xml" % (builddir),
        "%s/license.xml" % (releaseDir))

def copyMavenJarsToReleaseDir(releaseDir, version):
    #The .jars and upload file must be in a directory called voltdb - it is the projectname
    mavenProjectDir = releaseDir + "/mavenjars/voltdb"
    if not os.path.exists(mavenProjectDir):
        os.makedirs(mavenProjectDir)

    #Get the voltdbclient-n.n.jar from the recently built community build
    get("%s/voltdb/obj/release/dist-client-java/voltdb/voltdbclient-%s.jar" % (builddir, version),
        "%s/voltdbclient-%s.jar" % (mavenProjectDir, version))
    #Get the upload.gradle file
    get("%s/voltdb/tools/kit_tools/upload.gradle" % (builddir),
        "%s/upload.gradle" % (mavenProjectDir))
    #Get the src and javadoc .jar files
    get("%s/voltdb/obj/release/voltdbclient-%s-javadoc.jar" % (builddir, version),
        "%s/voltdbclient-%s-javadoc.jar" % (mavenProjectDir, version))
    get("%s/voltdb/obj/release/voltdbclient-%s-sources.jar" % (builddir, version),
        "%s/voltdbclient-%s-sources.jar" % (mavenProjectDir, version))

################################################
# CREATE CANDIDATE SYMLINKS
################################################

def createCandidateSysmlink(releaseDir):
    candidateDir =  os.getenv('HOME') + "/releases/candidate";
    local("rm -rf " + candidateDir)
    local("ln -s %s %s" % (releaseDir, candidateDir))

################################################
# BACKUP RELEASE DIR
################################################

def backupReleaseDir(releaseDir,archiveDir,version):
    if not os.path.exists(archiveDir):
        os.makedirs(archiveDir)
    # make a backup with the timstamp of the  build
    timestamp = datetime.datetime.now().strftime("%y%m%d-%H%M%S")
    local("tar -czf %s/%s-%s.tgz %s" \
          % (archiveDir, version, timestamp, releaseDir))

################################################
# REMOVE NATIVE LIBS FROM SHARED DIRECTORY
################################################

def rmNativeLibs():
    # local("ls -l ~" + username + nativelibdir)
    local("rm -rf ~" + username + nativelibdir)

################################################
# GET THE GIT TAGS OR SHAS TO BUILD FROM
################################################

parser = argparse.ArgumentParser(description = "Create a full kit. With no args, will do build of master")
parser.add_argument('voltdb_sha', nargs="?", default="master", help="voltdb repository commit, tag or branch" )
parser.add_argument('pro_sha', nargs="?", default="master", help="pro repository commit, tag or branch" )
parser.add_argument('rabbitmq_sha', nargs="?", default="master", help="rabbitmq repository commit, tag or branch" )
parser.add_argument('-g','--gitloc', default="git@github.com:VoltDB", help="Repository location. For example: /home/github-mirror")
parser.add_argument('--nomac', action='store_true', help="Don't build Mac OSX")
parser.add_argument('--nocommunity', action='store_true', help="Don't build community")
args = parser.parse_args()

proTreeish = args.pro_sha
voltdbTreeish = args.voltdb_sha
rbmqExportTreeish = args.rabbitmq_sha

print args

build_community = not args.nocommunity
build_mac = not args.nomac

#If anything is missing we're going to dump this in oneoffs dir.
build_all = build_community and build_mac
if voltdbTreeish != proTreeish or not build_all:
    oneOff = True
else:
    oneOff  = False

rmNativeLibs()

try:
    build_args = os.environ['VOLTDB_BUILD_ARGS']
except:
    build_args=""

print "Building with pro: %s and voltdb: %s" % (proTreeish, voltdbTreeish)

build_errors=False

versionCentos = "unknown"
versionMac = "unknown"
releaseDir = "unknown"

# get ssh config [key_filename, hostname]
CentosSSHInfo = getSSHInfoForHost("volt15a")
MacSSHInfo = getSSHInfoForHost("voltmini")
UbuntuSSHInfo = getSSHInfoForHost("volt12d")

# build community kit on the mini so that .so can be picked up for unified kit
if build_mac or build_community:
    try:
        with settings(user=username,host_string=MacSSHInfo[1],disable_known_hosts=True,key_filename=MacSSHInfo[0]):
            versionMac = checkoutCode(voltdbTreeish, proTreeish, rbmqExportTreeish, args.gitloc)
            buildCommunity()
    except Exception as e:
        print traceback.format_exc()
        print "Could not build MAC kit. Exception: " + str(e) + ", Type: " + str(type(e))
        build_errors=True

# build kits on 15f
try:
    with settings(user=username,host_string=CentosSSHInfo[1],disable_known_hosts=True,key_filename=CentosSSHInfo[0]):
        versionCentos = checkoutCode(voltdbTreeish, proTreeish, rbmqExportTreeish, args.gitloc)
        if build_mac:
            assert versionCentos == versionMac

        if oneOff:
            releaseDir = "%s/releases/one-offs/%s-%s-%s" % \
                (os.getenv('HOME'), versionCentos, voltdbTreeish, proTreeish)
        else:
            releaseDir = os.getenv('HOME') + "/releases/" + voltdbTreeish
        makeReleaseDir(releaseDir)
        print "VERSION: " + versionCentos
        if build_community:
            buildCommunity()
            buildRabbitMQExport(versionCentos, "community")
            copyCommunityFilesToReleaseDir(releaseDir, versionCentos, "LINUX")
        buildEnterprise()
        buildRabbitMQExport(versionCentos, "ent")
        makeSHA256SUM(versionCentos,"ent")
        copyFilesToReleaseDir(releaseDir, versionCentos, "ent")
        packagePro(versionCentos)
        makeSHA256SUM(versionCentos,"pro")
        copyFilesToReleaseDir(releaseDir, versionCentos, "pro")
        makeTrialLicense()
        copyTrialLicenseToReleaseDir(releaseDir)
        makeMavenJars()
        copyMavenJarsToReleaseDir(releaseDir, versionCentos)

except Exception as e:
    print traceback.format_exc()
    print "Could not build LINUX kit. Exception: " + str(e) + ", Type: " + str(type(e))
    build_errors=True

rmNativeLibs()      # cleanup imported native libs so not picked up unexpectedly by other builds

exit (build_errors)
#archiveDir = os.path.join(os.getenv('HOME'), "releases", "archive", voltdbTreeish, versionCentos)
#backupReleaseDir(releaseDir, archiveDir, versionCentos)
<commit_msg>build_kit.py: rabbitmq is now mirrored<commit_after>#!/usr/bin/env python

import argparse, datetime, getpass, os, sys, shutil, traceback
from fabric.api import run, cd, local, get, settings, lcd, put
from fabric_ssh_config import getSSHInfoForHost
from fabric.context_managers import shell_env
from fabric.utils import abort

#Login as user test, but build in a directory by real username
username = 'test'
builddir = "/tmp/" + getpass.getuser() + "Kits/buildtemp"
version = "UNKNOWN"
nativelibdir = "/nativelibs/obj"  #  ~test/libs/... usually
defaultlicensedays = 70 #default trial license length

################################################
# CHECKOUT CODE INTO A TEMP DIR
################################################

def checkoutCode(voltdbGit, proGit, rbmqExportGit, gitloc):
    global buildir
    # clean out the existing dir
    run("rm -rf " + builddir)
    # make the build dir again
    run("mkdir -p " + builddir)
    # change to it
    with cd(builddir):
        # do the checkouts, collect checkout errors on both community &
        # pro repos so user gets status on both checkouts
        message = ""
        run("git clone -q %s/voltdb.git" % gitloc)
        result = run("cd voltdb; git checkout %s" % voltdbGit, warn_only=True)
        if result.failed:
            message = "VoltDB checkout failed. Missing branch %s." % rbmqExportGit

        run("git clone -q %s/pro.git" % gitloc)
        result = run("cd pro; git checkout %s" % proGit, warn_only=True)
        if result.failed:
            message += "\nPro checkout failed. Missing branch %s." % rbmqExportGit

        run("git clone -q %s/export-rabbitmq.git" % gitloc)
        result = run("cd export-rabbitmq; git checkout %s" % rbmqExportGit, warn_only=True)
        # Probably ok to use master for export-rabbitmq.
        if result.failed:
            print "\nExport-rabbitmg branch %s checkout failed. Defaulting to master." % rbmqExportGit

        if len(message) > 0:
            abort(message)

        return run("cat voltdb/version.txt").strip()

################################################
# MAKE A RELEASE DIR
################################################

def makeReleaseDir(releaseDir):
    # handle the case where a release dir exists for this version
    if os.path.exists(releaseDir):
        shutil.rmtree(releaseDir)
    # create a release dir
    os.makedirs(releaseDir)
    print "Created dir: " + releaseDir

################################################
# BUILD THE COMMUNITY VERSION
################################################

def buildCommunity():
    if build_mac:
        packageMacLib="true"
    else:
        packageMacLib="false"
    with cd(builddir + "/voltdb"):
        run("pwd")
        run("git status")
        run("git describe --dirty")
        run("ant -Djmemcheck=NO_MEMCHECK -Dkitbuild=%s %s clean default dist" % (packageMacLib,  build_args))

################################################
# BUILD THE ENTERPRISE VERSION
################################################

def buildEnterprise():
    if build_mac:
        packageMacLib="true"
    else:
        packageMacLib="false"
    with cd(builddir + "/pro"):
        run("pwd")
        run("git status")
        run("git describe --dirty")
        run("VOLTCORE=../voltdb ant -f mmt.xml -Djmemcheck=NO_MEMCHECK -Dallowreplication=true -DallowDrActiveActive=true -Dlicensedays=%d -Dkitbuild=%s %s clean dist.pro" % (defaultlicensedays, packageMacLib, build_args))

################################################
# BUILD THE PRO VERSION
################################################

#
def packagePro(version):
    print "Making license"
    makeTrialLicense(days=defaultlicensedays, dr_and_xdcr=False, nodes=3)
    print "Repacking pro kit"
    with cd(builddir + "/pro/obj/pro"):
        run("mkdir pro_kit_staging")
    with cd(builddir + "/pro/obj/pro/pro_kit_staging"):
        run("tar xf ../voltdb-ent-%s.tar.gz" % version)
        run("mv voltdb-ent-%s voltdb-pro-%s" % (version, version))
        run("cp %s/pro/trial_*.xml voltdb-pro-%s/voltdb/license.xml" % (builddir, version))
        run("tar cvfz ../voltdb-pro-%s.tar.gz voltdb-pro-%s" % (version, version))

################################################
# BUILD THE RABBITMQ EXPORT CONNECTOR
################################################
#Build rabbit MQ Exporter
def buildRabbitMQExport(version, dist_type):
    # Paths to the final kit for unpacking/repacking with rmq export
    paths = {
        'community': builddir + "/voltdb/obj/release",
        'ent' : builddir + "/pro/obj/pro/"
        }
    # Untar
    with cd(paths[dist_type]):
        run ("pwd")
        run ("mkdir -p restage")
        run ("tar xf voltdb-%s-%s.tar.gz -C restage" % (dist_type, version))
        run ("rm -f voltdb-%s-%s.tar.gz" % (dist_type, version))

    # Build RabbitMQ export jar and put it into the untarred kit
    with cd(builddir + "/export-rabbitmq"):
        run("pwd")
        run("git status")
        run("git describe --dirty", warn_only=True)
        run("VOLTDIST=%s/restage/voltdb-%s-%s ant" % (paths[dist_type], dist_type, version))

    # Retar
    with cd(paths[dist_type]):
        run("pwd")
        run("tar -C restage -czf voltdb-%s-%s.tar.gz voltdb-%s-%s" % (dist_type, version, dist_type, version))
        run ("rm -Rf restage")

################################################
# MAKE AN ENTERPRISE TRIAL LICENSE
################################################

# Must be called after buildEnterprise has been done
def makeTrialLicense(days=30, dr_and_xdcr="true", nodes=12):
    with cd(builddir + "/pro/tools"):
        run("./make_trial_licenses.pl -t %d -H %d -W %s" % (days, nodes, dr_and_xdcr ))

################################################
# MAKE A SHA256 checksum
################################################

def makeSHA256SUM(version, type):
    with cd(builddir + "/pro/obj/pro"):
        kitname="voltdb-" +  type + "-" + version
        run("sha256sum -b %s.tar.gz > %s.SHA256SUM" % (kitname, kitname))

################################################
# MAKE AN JAR FILES NEEDED TO PUSH TO MAVEN
################################################

def makeMavenJars():
    with cd(builddir + "/voltdb"):
        run("VOLTCORE=../voltdb ant -f build-client.xml maven-jars")

################################################
# COPY FILES
################################################

def copyFilesToReleaseDir(releaseDir, version, type=None):
    print "Copying files to releaseDir"
    if type:
        typeString="-" + type
    else:
        typeString=""
    get("%s/pro/obj/pro/voltdb%s-%s.tar.gz" % (builddir, typeString, version),
        "%s/voltdb%s-%s.tar.gz" % (releaseDir, typeString, version))
    get("%s/pro/obj/pro/voltdb%s-%s.SHA256SUM" % (builddir, typeString, version),
        "%s/voltdb%s-%s.SHA256SUM" % (releaseDir, typeString, version))

def copyCommunityFilesToReleaseDir(releaseDir, version, operatingsys):
    get("%s/voltdb/obj/release/voltdb-community-%s.tar.gz" % (builddir, version),
        "%s/voltdb-community-%s.tar.gz" % (releaseDir, version))

    # add stripped symbols
    if operatingsys == "LINUX":
        os.makedirs(releaseDir + "/other")
        get("%s/voltdb/obj/release/voltdb-%s.sym" % (builddir, version),
            "%s/other/%s-voltdb-voltkv-%s.sym" % (releaseDir, operatingsys, version))

def copyTrialLicenseToReleaseDir(releaseDir):
    get("%s/pro/trial_*.xml" % (builddir),
        "%s/license.xml" % (releaseDir))

def copyMavenJarsToReleaseDir(releaseDir, version):
    #The .jars and upload file must be in a directory called voltdb - it is the projectname
    mavenProjectDir = releaseDir + "/mavenjars/voltdb"
    if not os.path.exists(mavenProjectDir):
        os.makedirs(mavenProjectDir)

    #Get the voltdbclient-n.n.jar from the recently built community build
    get("%s/voltdb/obj/release/dist-client-java/voltdb/voltdbclient-%s.jar" % (builddir, version),
        "%s/voltdbclient-%s.jar" % (mavenProjectDir, version))
    #Get the upload.gradle file
    get("%s/voltdb/tools/kit_tools/upload.gradle" % (builddir),
        "%s/upload.gradle" % (mavenProjectDir))
    #Get the src and javadoc .jar files
    get("%s/voltdb/obj/release/voltdbclient-%s-javadoc.jar" % (builddir, version),
        "%s/voltdbclient-%s-javadoc.jar" % (mavenProjectDir, version))
    get("%s/voltdb/obj/release/voltdbclient-%s-sources.jar" % (builddir, version),
        "%s/voltdbclient-%s-sources.jar" % (mavenProjectDir, version))

################################################
# CREATE CANDIDATE SYMLINKS
################################################

def createCandidateSysmlink(releaseDir):
    candidateDir =  os.getenv('HOME') + "/releases/candidate";
    local("rm -rf " + candidateDir)
    local("ln -s %s %s" % (releaseDir, candidateDir))

################################################
# BACKUP RELEASE DIR
################################################

def backupReleaseDir(releaseDir,archiveDir,version):
    if not os.path.exists(archiveDir):
        os.makedirs(archiveDir)
    # make a backup with the timstamp of the  build
    timestamp = datetime.datetime.now().strftime("%y%m%d-%H%M%S")
    local("tar -czf %s/%s-%s.tgz %s" \
          % (archiveDir, version, timestamp, releaseDir))

################################################
# REMOVE NATIVE LIBS FROM SHARED DIRECTORY
################################################

def rmNativeLibs():
    # local("ls -l ~" + username + nativelibdir)
    local("rm -rf ~" + username + nativelibdir)

################################################
# GET THE GIT TAGS OR SHAS TO BUILD FROM
################################################

parser = argparse.ArgumentParser(description = "Create a full kit. With no args, will do build of master")
parser.add_argument('voltdb_sha', nargs="?", default="master", help="voltdb repository commit, tag or branch" )
parser.add_argument('pro_sha', nargs="?", default="master", help="pro repository commit, tag or branch" )
parser.add_argument('rabbitmq_sha', nargs="?", default="master", help="rabbitmq repository commit, tag or branch" )
parser.add_argument('-g','--gitloc', default="git@github.com:VoltDB", help="Repository location. For example: /home/github-mirror")
parser.add_argument('--nomac', action='store_true', help="Don't build Mac OSX")
parser.add_argument('--nocommunity', action='store_true', help="Don't build community")
args = parser.parse_args()

proTreeish = args.pro_sha
voltdbTreeish = args.voltdb_sha
rbmqExportTreeish = args.rabbitmq_sha

print args

build_community = not args.nocommunity
build_mac = not args.nomac

#If anything is missing we're going to dump this in oneoffs dir.
build_all = build_community and build_mac
if voltdbTreeish != proTreeish or not build_all:
    oneOff = True
else:
    oneOff  = False

rmNativeLibs()

try:
    build_args = os.environ['VOLTDB_BUILD_ARGS']
except:
    build_args=""

print "Building with pro: %s and voltdb: %s" % (proTreeish, voltdbTreeish)

build_errors=False

versionCentos = "unknown"
versionMac = "unknown"
releaseDir = "unknown"

# get ssh config [key_filename, hostname]
CentosSSHInfo = getSSHInfoForHost("volt15a")
MacSSHInfo = getSSHInfoForHost("voltmini")
UbuntuSSHInfo = getSSHInfoForHost("volt12d")

# build community kit on the mini so that .so can be picked up for unified kit
if build_mac or build_community:
    try:
        with settings(user=username,host_string=MacSSHInfo[1],disable_known_hosts=True,key_filename=MacSSHInfo[0]):
            versionMac = checkoutCode(voltdbTreeish, proTreeish, rbmqExportTreeish, args.gitloc)
            buildCommunity()
    except Exception as e:
        print traceback.format_exc()
        print "Could not build MAC kit. Exception: " + str(e) + ", Type: " + str(type(e))
        build_errors=True

# build kits on 15f
try:
    with settings(user=username,host_string=CentosSSHInfo[1],disable_known_hosts=True,key_filename=CentosSSHInfo[0]):
        versionCentos = checkoutCode(voltdbTreeish, proTreeish, rbmqExportTreeish, args.gitloc)
        if build_mac:
            assert versionCentos == versionMac

        if oneOff:
            releaseDir = "%s/releases/one-offs/%s-%s-%s" % \
                (os.getenv('HOME'), versionCentos, voltdbTreeish, proTreeish)
        else:
            releaseDir = os.getenv('HOME') + "/releases/" + voltdbTreeish
        makeReleaseDir(releaseDir)
        print "VERSION: " + versionCentos
        if build_community:
            buildCommunity()
            buildRabbitMQExport(versionCentos, "community")
            copyCommunityFilesToReleaseDir(releaseDir, versionCentos, "LINUX")
        buildEnterprise()
        buildRabbitMQExport(versionCentos, "ent")
        makeSHA256SUM(versionCentos,"ent")
        copyFilesToReleaseDir(releaseDir, versionCentos, "ent")
        packagePro(versionCentos)
        makeSHA256SUM(versionCentos,"pro")
        copyFilesToReleaseDir(releaseDir, versionCentos, "pro")
        makeTrialLicense()
        copyTrialLicenseToReleaseDir(releaseDir)
        makeMavenJars()
        copyMavenJarsToReleaseDir(releaseDir, versionCentos)

except Exception as e:
    print traceback.format_exc()
    print "Could not build LINUX kit. Exception: " + str(e) + ", Type: " + str(type(e))
    build_errors=True

rmNativeLibs()      # cleanup imported native libs so not picked up unexpectedly by other builds

exit (build_errors)
#archiveDir = os.path.join(os.getenv('HOME'), "releases", "archive", voltdbTreeish, versionCentos)
#backupReleaseDir(releaseDir, archiveDir, versionCentos)
<|endoftext|>


<commit_before>#!/usr/bin/env python
# encoding: utf-8

# The MIT License (MIT)

# Copyright (c) 2018 CNRS

# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:

# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.

# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.

# AUTHORS
# Hervé BREDIN - http://herve.niderb.fr

import numpy as np
from tqdm import tqdm
import torch
from torch.autograd import Variable
import torch.nn.functional as F
from pyannote.audio.generators.speaker import SpeechSegmentGenerator
from pyannote.audio.callback import LoggingCallbackPytorch
from torch.optim import Adam
from scipy.spatial.distance import pdist
from .triplet_loss import TripletLoss

class WTFTripletLoss(TripletLoss):
    """

    Parameters
    ----------
    duration : float, optional
        Defautls to 3.2 seconds.
    margin: float, optional
        Margin factor. Defaults to 0.2.
    sampling : {'all', 'hard'}, optional
        Triplet sampling strategy.
    per_label : int, optional
        Number of sequences per speaker in each batch. Defaults to 3.
    per_fold : int, optional
        If provided, sample triplets from groups of `per_fold` speakers at a
        time. Defaults to sample triplets from the whole speaker set.
    parallel : int, optional
        Number of prefetching background generators. Defaults to 1.
        Each generator will prefetch enough batches to cover a whole epoch.
        Set `parallel` to 0 to not use background generators.
    """

    def __init__(self, duration=3.2, sampling='all',
                 per_label=3, per_fold=None, parallel=1):

        super(WTFTripletLoss, self).__init__(
            duration=duration, metric='angular', clamp='sigmoid',
            sampling=sampling, per_label=per_label, per_fold=per_fold,
            parallel=parallel)

    def fit(self, model, feature_extraction, protocol, log_dir, subset='train',
            epochs=1000, restart=None, gpu=False):

        import tensorboardX
        writer = tensorboardX.SummaryWriter(log_dir=log_dir)

        logging_callback = LoggingCallbackPytorch(
            log_dir=log_dir, restart=(False if restart is None else True))

        batch_generator = SpeechSegmentGenerator(
            feature_extraction,
            per_label=self.per_label, per_fold=self.per_fold,
            duration=self.duration, parallel=self.parallel)
        batches = batch_generator(protocol, subset=subset)
        batch = next(batches)

        batches_per_epoch = batch_generator.batches_per_epoch

        if restart is not None:
            weights_pt = logging_callback.WEIGHTS_PT.format(
                log_dir=log_dir, epoch=restart)
            model.load_state_dict(torch.load(weights_pt))

        if gpu:
            model = model.cuda()

        model.internal = False

        optimizer = Adam(model.parameters())
        if restart is not None:
            optimizer_pt = logging_callback.OPTIMIZER_PT.format(
                log_dir=log_dir, epoch=restart)
            optimizer.load_state_dict(torch.load(optimizer_pt))
            if gpu:
                for state in optimizer.state.values():
                    for k, v in state.items():
                        if torch.is_tensor(v):
                            state[k] = v.cuda()

        restart = 0 if restart is None else restart + 1
        for epoch in range(restart, restart + epochs):

            tloss_avg, closs_avg = 0., 0.

            if epoch % 10:
                positive, negative = [], []
                norms = []

            desc = 'Epoch #{0}'.format(epoch)
            for i in tqdm(range(batches_per_epoch), desc=desc):

                model.zero_grad()

                batch = next(batches)

                X = Variable(torch.from_numpy(
                    np.array(np.rollaxis(batch['X'], 0, 2),
                             dtype=np.float32)))

                if gpu:
                    X = X.cuda()

                fX = model(X)

                if epoch % 10 == 0:
                    if gpu:
                        fX_ = fX.data.cpu().numpy()
                    else:
                        fX_ = fX.data.numpy()
                    norms.append(np.linalg.norm(fX_, axis=1))

                # pre-compute pairwise distances
                distances = self.pdist(fX)

                if epoch % 10 == 0:
                    if gpu:
                        distances_ = distances.data.cpu().numpy()
                    else:
                        distances_ = distances.data.numpy()
                    is_positive = pdist(batch['y'].reshape((-1, 1)), metric='chebyshev') < 1
                    positive.append(distances_[np.where(is_positive)])
                    negative.append(distances_[np.where(~is_positive)])

                # sample triplets
                if self.sampling == 'all':
                    anchors, positives, negatives = self.batch_all(
                        batch['y'], distances)

                elif self.sampling == 'hard':
                    anchors, positives, negatives = self.batch_hard(
                        batch['y'], distances)

                # compute triplet loss
                tlosses, deltas = self.triplet_loss(
                    distances, anchors, positives, negatives,
                    return_delta=True)

                tloss = torch.mean(tlosses)

                # compute wtf loss
                closses = F.sigmoid(
                    F.softsign(deltas) * torch.norm(fX[anchors], 2, 1, keepdim=True))
                closs = torch.mean(closses)

                # log loss
                if gpu:
                    tloss_ = float(tloss.data.cpu().numpy())
                    closs_ = float(closs.data.cpu().numpy())
                else:
                    tloss_ = float(tloss.data.numpy())
                    closs_ = float(closs.data.numpy())
                tloss_avg += tloss_
                closs_avg += closs_

                loss = tloss + closs
                loss.backward()
                optimizer.step()

            tloss_avg /= batches_per_epoch
            writer.add_scalar('tloss', tloss_avg, global_step=epoch)

            closs_avg /= batches_per_epoch
            writer.add_scalar('closs', closs_avg, global_step=epoch)

            if epoch % 10 == 0:
                positive = np.hstack(positive)
                negative = np.hstack(negative)
                writer.add_histogram(
                    'embedding/pairwise_distance/positive', positive,
                    global_step=epoch, bins='auto')
                writer.add_histogram(
                    'embedding/pairwise_distance/negative', negative,
                    global_step=epoch, bins='auto')

                norms = np.hstack(norms)
                writer.add_histogram(
                    'embedding/norm', norms,
                    global_step=epoch, bins='auto')

            logging_callback.model = model
            logging_callback.optimizer = optimizer
            logging_callback.on_epoch_end(epoch)
<commit_msg>fix: fix typo<commit_after>#!/usr/bin/env python
# encoding: utf-8

# The MIT License (MIT)

# Copyright (c) 2018 CNRS

# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:

# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.

# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.

# AUTHORS
# Hervé BREDIN - http://herve.niderb.fr

import numpy as np
from tqdm import tqdm
import torch
from torch.autograd import Variable
import torch.nn.functional as F
from pyannote.audio.generators.speaker import SpeechSegmentGenerator
from pyannote.audio.callback import LoggingCallbackPytorch
from torch.optim import Adam
from scipy.spatial.distance import pdist
from .triplet_loss import TripletLoss

class WTFTripletLoss(TripletLoss):
    """

    Parameters
    ----------
    duration : float, optional
        Defautls to 3.2 seconds.
    margin: float, optional
        Margin factor. Defaults to 0.2.
    sampling : {'all', 'hard'}, optional
        Triplet sampling strategy.
    per_label : int, optional
        Number of sequences per speaker in each batch. Defaults to 3.
    per_fold : int, optional
        If provided, sample triplets from groups of `per_fold` speakers at a
        time. Defaults to sample triplets from the whole speaker set.
    parallel : int, optional
        Number of prefetching background generators. Defaults to 1.
        Each generator will prefetch enough batches to cover a whole epoch.
        Set `parallel` to 0 to not use background generators.
    """

    def __init__(self, duration=3.2, sampling='all',
                 per_label=3, per_fold=None, parallel=1):

        super(WTFTripletLoss, self).__init__(
            duration=duration, metric='angular', clamp='sigmoid',
            sampling=sampling, per_label=per_label, per_fold=per_fold,
            parallel=parallel)

    def fit(self, model, feature_extraction, protocol, log_dir, subset='train',
            epochs=1000, restart=None, gpu=False):

        import tensorboardX
        writer = tensorboardX.SummaryWriter(log_dir=log_dir)

        logging_callback = LoggingCallbackPytorch(
            log_dir=log_dir, restart=(False if restart is None else True))

        batch_generator = SpeechSegmentGenerator(
            feature_extraction,
            per_label=self.per_label, per_fold=self.per_fold,
            duration=self.duration, parallel=self.parallel)
        batches = batch_generator(protocol, subset=subset)
        batch = next(batches)

        batches_per_epoch = batch_generator.batches_per_epoch

        if restart is not None:
            weights_pt = logging_callback.WEIGHTS_PT.format(
                log_dir=log_dir, epoch=restart)
            model.load_state_dict(torch.load(weights_pt))

        if gpu:
            model = model.cuda()

        model.internal = False

        optimizer = Adam(model.parameters())
        if restart is not None:
            optimizer_pt = logging_callback.OPTIMIZER_PT.format(
                log_dir=log_dir, epoch=restart)
            optimizer.load_state_dict(torch.load(optimizer_pt))
            if gpu:
                for state in optimizer.state.values():
                    for k, v in state.items():
                        if torch.is_tensor(v):
                            state[k] = v.cuda()

        restart = 0 if restart is None else restart + 1
        for epoch in range(restart, restart + epochs):

            tloss_avg, closs_avg = 0., 0.

            if epoch % 10 == 0:
                positive, negative = [], []
                norms = []

            desc = 'Epoch #{0}'.format(epoch)
            for i in tqdm(range(batches_per_epoch), desc=desc):

                model.zero_grad()

                batch = next(batches)

                X = Variable(torch.from_numpy(
                    np.array(np.rollaxis(batch['X'], 0, 2),
                             dtype=np.float32)))

                if gpu:
                    X = X.cuda()

                fX = model(X)

                if epoch % 10 == 0:
                    if gpu:
                        fX_ = fX.data.cpu().numpy()
                    else:
                        fX_ = fX.data.numpy()
                    norms.append(np.linalg.norm(fX_, axis=1))

                # pre-compute pairwise distances
                distances = self.pdist(fX)

                if epoch % 10 == 0:
                    if gpu:
                        distances_ = distances.data.cpu().numpy()
                    else:
                        distances_ = distances.data.numpy()
                    is_positive = pdist(batch['y'].reshape((-1, 1)), metric='chebyshev') < 1
                    positive.append(distances_[np.where(is_positive)])
                    negative.append(distances_[np.where(~is_positive)])

                # sample triplets
                if self.sampling == 'all':
                    anchors, positives, negatives = self.batch_all(
                        batch['y'], distances)

                elif self.sampling == 'hard':
                    anchors, positives, negatives = self.batch_hard(
                        batch['y'], distances)

                # compute triplet loss
                tlosses, deltas = self.triplet_loss(
                    distances, anchors, positives, negatives,
                    return_delta=True)

                tloss = torch.mean(tlosses)

                # compute wtf loss
                closses = F.sigmoid(
                    F.softsign(deltas) * torch.norm(fX[anchors], 2, 1, keepdim=True))
                closs = torch.mean(closses)

                # log loss
                if gpu:
                    tloss_ = float(tloss.data.cpu().numpy())
                    closs_ = float(closs.data.cpu().numpy())
                else:
                    tloss_ = float(tloss.data.numpy())
                    closs_ = float(closs.data.numpy())
                tloss_avg += tloss_
                closs_avg += closs_

                loss = tloss + closs
                loss.backward()
                optimizer.step()

            tloss_avg /= batches_per_epoch
            writer.add_scalar('tloss', tloss_avg, global_step=epoch)

            closs_avg /= batches_per_epoch
            writer.add_scalar('closs', closs_avg, global_step=epoch)

            if epoch % 10 == 0:
                positive = np.hstack(positive)
                negative = np.hstack(negative)
                writer.add_histogram(
                    'embedding/pairwise_distance/positive', positive,
                    global_step=epoch, bins='auto')
                writer.add_histogram(
                    'embedding/pairwise_distance/negative', negative,
                    global_step=epoch, bins='auto')

                norms = np.hstack(norms)
                writer.add_histogram(
                    'embedding/norm', norms,
                    global_step=epoch, bins='auto')

            logging_callback.model = model
            logging_callback.optimizer = optimizer
            logging_callback.on_epoch_end(epoch)
<|endoftext|>


<commit_before>__authors__ = "Ian Goodfellow"
__copyright__ = "Copyright 2010-2012, Universite de Montreal"
__credits__ = ["Ian Goodfellow"]
__license__ = "3-clause BSD"
__maintainer__ = "Ian Goodfellow"
__email__ = "goodfeli@iro"

from pylearn2.testing.skip import skip_if_no_gpu
skip_if_no_gpu()

import numpy as np
from theano import shared
from pylearn2.sandbox.cuda_convnet.filter_acts import FilterActs
from pylearn2.sandbox.cuda_convnet.weight_acts import WeightActs
from theano.sandbox.cuda import gpu_from_host
from theano.sandbox.cuda import host_from_gpu
from theano.sandbox.rng_mrg import MRG_RandomStreams
import theano.tensor as T
from theano.tensor.nnet.conv import conv2d
from theano import function
import warnings

def test_match_grad_valid_conv():

    # Tests that weightActs is the gradient of FilterActs
    # with respect to the weights.

    rng = np.random.RandomState([2012,10,9])

    batch_size = 3
    rows = 10
    cols = 9
    channels = 6
    filter_rows = 4
    filter_cols = filter_rows
    num_filters = 16

    images = shared(rng.uniform(-1., 1., (channels, rows, cols,
        batch_size)).astype('float32'), name='images')
    filters = shared(rng.uniform(-1., 1., (channels, filter_rows,
        filter_cols, num_filters)).astype('float32'), name='filters')

    gpu_images = gpu_from_host(images)
    gpu_filters = gpu_from_host(filters)

    output = FilterActs()(gpu_images, gpu_filters)
    output = host_from_gpu(output)

    images_bc01 = images.dimshuffle(3,0,1,2)
    filters_bc01 = filters.dimshuffle(3,0,1,2)
    filters_bc01 = filters_bc01[:,:,::-1,::-1]

    output_conv2d = conv2d(images_bc01, filters_bc01,
            border_mode='valid')

    output_conv2d = output_conv2d.dimshuffle(1,2,3,0)

    theano_rng = MRG_RandomStreams(2013 + 1 + 31)

    coeffs = theano_rng.normal(avg=0., std=1., size=output_conv2d.shape, dtype='float32')

    cost_conv2d = (coeffs * output_conv2d).sum()

    weights_grad_conv2d = T.grad(cost_conv2d, filters)

    coeffs = coeffs.dimshuffle(1, 2, 3, 0)
    cost = (coeffs * output).sum()
    hid_acts_grad = T.grad(cost, output)

    weights_grad = WeightActs()(gpu_images, gpu_from_host(hid_acts_grad))

    f = function([], [output, output_conv2d, weights_grad, weights_grad_conv2d])

    output, output_conv2d = f()

    warnings.warn("""test_match_valid_conv success criterion is not very strict. Can we verify that this is OK?
                     One possibility is that theano is numerically unstable and Alex's code is better.
                     Probably theano CPU 64 bit is OK but it's worth checking the others.""")
    if np.abs(output - output_conv2d).max() > 2.4e-6:
        assert type(output) == type(output_conv2d)
        assert output.dtype == output_conv2d.dtype
        if output.shape != output_conv2d.shape:
            print 'cuda-convnet shape: ',output.shape
            print 'theano shape: ',output_conv2d.shape
            assert False
        err = np.abs(output - output_conv2d)
        print 'absolute error range: ', (err.min(), err.max())
        print 'mean absolute error: ', err.mean()
        print 'cuda-convnet value range: ', (output.min(), output.max())
        print 'theano value range: ', (output_conv2d.min(), output_conv2d.max())
        assert False

if __name__ == '__main__':
    test_match_grad_valid_conv()

<commit_msg>removed an extra dimshuffle<commit_after>__authors__ = "Ian Goodfellow"
__copyright__ = "Copyright 2010-2012, Universite de Montreal"
__credits__ = ["Ian Goodfellow"]
__license__ = "3-clause BSD"
__maintainer__ = "Ian Goodfellow"
__email__ = "goodfeli@iro"

from pylearn2.testing.skip import skip_if_no_gpu
skip_if_no_gpu()

import numpy as np
from theano import shared
from pylearn2.sandbox.cuda_convnet.filter_acts import FilterActs
from pylearn2.sandbox.cuda_convnet.weight_acts import WeightActs
from theano.sandbox.cuda import gpu_from_host
from theano.sandbox.cuda import host_from_gpu
from theano.sandbox.rng_mrg import MRG_RandomStreams
import theano.tensor as T
from theano.tensor.nnet.conv import conv2d
from theano import function
import warnings

def test_match_grad_valid_conv():

    # Tests that weightActs is the gradient of FilterActs
    # with respect to the weights.

    rng = np.random.RandomState([2012,10,9])

    batch_size = 3
    rows = 10
    cols = 9
    channels = 6
    filter_rows = 4
    filter_cols = filter_rows
    num_filters = 16

    images = shared(rng.uniform(-1., 1., (channels, rows, cols,
        batch_size)).astype('float32'), name='images')
    filters = shared(rng.uniform(-1., 1., (channels, filter_rows,
        filter_cols, num_filters)).astype('float32'), name='filters')

    gpu_images = gpu_from_host(images)
    gpu_filters = gpu_from_host(filters)

    output = FilterActs()(gpu_images, gpu_filters)
    output = host_from_gpu(output)

    images_bc01 = images.dimshuffle(3,0,1,2)
    filters_bc01 = filters.dimshuffle(3,0,1,2)
    filters_bc01 = filters_bc01[:,:,::-1,::-1]

    output_conv2d = conv2d(images_bc01, filters_bc01,
            border_mode='valid')

    output_conv2d = output_conv2d.dimshuffle(1,2,3,0)

    theano_rng = MRG_RandomStreams(2013 + 1 + 31)

    coeffs = theano_rng.normal(avg=0., std=1., size=output_conv2d.shape, dtype='float32')

    cost_conv2d = (coeffs * output_conv2d).sum()

    weights_grad_conv2d = T.grad(cost_conv2d, filters)

    cost = (coeffs * output).sum()
    hid_acts_grad = T.grad(cost, output)

    weights_grad = WeightActs()(gpu_images, gpu_from_host(hid_acts_grad))

    f = function([], [output, output_conv2d, weights_grad, weights_grad_conv2d])

    output, output_conv2d = f()

    warnings.warn("""test_match_valid_conv success criterion is not very strict. Can we verify that this is OK?
                     One possibility is that theano is numerically unstable and Alex's code is better.
                     Probably theano CPU 64 bit is OK but it's worth checking the others.""")
    if np.abs(output - output_conv2d).max() > 2.4e-6:
        assert type(output) == type(output_conv2d)
        assert output.dtype == output_conv2d.dtype
        if output.shape != output_conv2d.shape:
            print 'cuda-convnet shape: ',output.shape
            print 'theano shape: ',output_conv2d.shape
            assert False
        err = np.abs(output - output_conv2d)
        print 'absolute error range: ', (err.min(), err.max())
        print 'mean absolute error: ', err.mean()
        print 'cuda-convnet value range: ', (output.min(), output.max())
        print 'theano value range: ', (output_conv2d.min(), output_conv2d.max())
        assert False

if __name__ == '__main__':
    test_match_grad_valid_conv()

<|endoftext|>


<commit_before>"""
    lstm (needs modularisation and further refactoring)

    sequence classification of variable lengths through
    Long-short Term Memory (LSTM) which models surrounding context of words.

    Words are represented in 300-dimension vectors to allow generalization over
    semantic redundancy.

    Distributed Representations of Words and Phrases and their Compositionality
    Mikolov et al. 2013
"""
import os
import random

import tensorflow as tf
import numpy as np

from tqdm import tqdm
from functools import wraps
from sklearn import model_selection, preprocessing

from chatbot.nlp.embedding import WordEmbedding
from chatbot.serializers import feed_conversation

from . import corpus, labels

os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2'

def flatten_split_resample(encoded_corpuses, encoded_labels,
                           valid_ratio=.2,
                           sample_size=1000):
    """break documents into sentences and augment, and one-hot encode labels"""

    def resample(docs, labels, sample_size=sample_size):
        unique_labels = np.unique(labels)
        indice_array = list()
        for label in unique_labels:
            indice_label = np.where(labels == label)[0].tolist()
            indice_array.extend(random.choices(indice_label, k=sample_size))
        return (docs[indice_array], labels[indice_array])

    flattened_docs = list()
    flattened_labels = list()
    for d, l in zip(encoded_corpuses, encoded_labels):
        for sent in d:
            flattened_docs.append(sent)
            flattened_labels.append(l)

    ravelled_corpus = np.array(flattened_docs)
    ravelled_labels = np.array(flattened_labels)

    X_train, X_test, y_train, y_test = \
        model_selection.train_test_split(ravelled_corpus, ravelled_labels,
                                         test_size=valid_ratio,
                                         stratify=ravelled_labels)

    return resample(X_train, y_train), resample(X_test, y_test)

def size(sequence):
    """produces length vector in shape [batch_size, length] for each padded
    sequence of its true length excluding zero vectors.
    sequence of vectors comes in shape of [batch_size, STEP_SIZE, dimensions]
    """
    flag = tf.sign(tf.reduce_max(tf.abs(sequence), axis=2))
    length = tf.reduce_sum(flag, axis=1)
    return tf.cast(length, tf.int32)

def find_last(outputs, length):
    """zero padded sequences will produce zero vectors after sequence_length
    required using dynamic unrolling dynamic_rnn API. this function locates
    the last non-zero output for calculating logits
    """
    # outputs in shape [BATCH_SIZE, STEP_SIZE, STATE_SIZE], length [BATCH_SIZE]
    b, t, st = tf.unstack(tf.shape(outputs))
    flat_index = tf.range(b) * t + (length - 1)
    zeros = tf.zeros(shape=[b], dtype=tf.int32)
    non_neg_flat_index = tf.maximum(zeros, flat_index)
    flat_outputs = tf.reshape(outputs, [-1, st])
    last_outputs = tf.gather(flat_outputs, non_neg_flat_index)
    return last_outputs

def timeit(func):
    """calculate time for a function to complete"""
    import time

    def wrapper(*args, **kwargs):
        start = time.time()
        output = func(*args, **kwargs)
        end = time.time()
        print('function {0} took {1:0.3f} s'.format(
              func.__name__, (end - start) * 1))
        return output
    return wrapper

def multithreading(func):
    """decorator using tensorflow threading ability."""
    @wraps(func)
    def wrapper(*args, **kwargs):
        coord = tf.train.Coordinator()
        threads = tf.train.start_queue_runners(coord=coord)
        func_output = func(*args, **kwargs)
        try:
            coord.request_stop()
            coord.join(threads, stop_grace_period_secs=5)
        except (tf.errors.CancelledError, RuntimeError) as e:
            pass
        return func_output
    return wrapper

def enqueue(sent_encoded, label_encoded, num_epochs=None, shuffle=True):
    """returns an Ops Tensor with queued sentence and label pair"""
    sent = tf.convert_to_tensor(sent_encoded, dtype=tf.int32)
    label = tf.convert_to_tensor(label_encoded, dtype=tf.uint8)
    input_queue = tf.train.slice_input_producer(
                                tensor_list=[sent, label],
                                num_epochs=num_epochs,
                                shuffle=shuffle)
    return input_queue

def batch_generator(sent_queue, label_queue, batch_size=None, threads=4):
    return tf.train.shuffle_batch(
                    tensors=[sent_queue, label_queue],
                    batch_size=batch_size,
                    num_threads=threads,
                    capacity=1e3,
                    min_after_dequeue=batch_size,
                    allow_smaller_final_batch=True)

@multithreading
def train(n, x, y_, train_sent_batch, train_label_batch, valid_sent_batch,
          valid_label_batch, keep_prob, optimiser, metric, loss, verbose):

    for global_step in tqdm(range(n), miniters=1):
        train_sent, train_label = \
                    sess.run(fetches=[train_sent_batch, train_label_batch])
        _, train_accuracy, train_loss = \
            sess.run(fetches=[optimiser, metric, loss],
                     feed_dict={x: train_sent, y_: train_label, keep_prob: .8})

        if verbose:
            print("step {0} of {3}, train accuracy: {1:.4f} log loss: {2:.4f}"
                  .format(global_step, train_accuracy, train_loss, n))

        if global_step and global_step % 10 == 0:
            valid_sent, valid_label = \
                    sess.run(fetches=[valid_sent_batch, valid_label_batch])
            valid_accuracy, valid_loss = sess.run(
                    fetches=[metric, loss],
                    feed_dict={x: valid_sent, y_: valid_label, keep_prob: 1})

            if verbose:
                print("step {0} of {3}, valid accuracy: {1:.4f} "
                      "log loss: {2:.4f}".format(global_step, valid_accuracy,
                                                 valid_loss, n))

N_DOC = 20
corpus = corpus[:N_DOC]
labels = labels[:N_DOC]

STATE_SIZE = 24
STEP_SIZE = 30
N_CLASS = len(labels)
BATCH_SIZE = 50
EPOCH = 200

corpus_encoder = WordEmbedding(top=None).fit(corpus)
encoded_corpus = corpus_encoder.encode(zero_pad=True, pad_length=STEP_SIZE)

l_encoder = preprocessing.LabelBinarizer().fit(labels)
encoded_labels, classes = l_encoder.transform(labels), l_encoder.classes_

embedding_matrix = corpus_encoder.vectorize()
embed_shape = embedding_matrix.shape

x = tf.placeholder(dtype=tf.int32, shape=(None, STEP_SIZE), name='feature')
y_ = tf.placeholder(dtype=tf.uint8, shape=(None, N_CLASS), name='label')
v = tf.placeholder(dtype=tf.float32, shape=(None, 300), name='vectors')
keep_prob = tf.placeholder(dtype=tf.float32, name='keep_rate')

embeddings = tf.get_variable(name='embedd', shape=embed_shape, trainable=False)

rnn_inputs = tf.nn.embedding_lookup(embeddings, x)

with tf.variable_scope('softmax'):
    W_softmax = tf.get_variable(
                    name='W',
                    shape=[STATE_SIZE, N_CLASS],
                    initializer=tf.truncated_normal_initializer(stddev=0.1))
    b_softmax = tf.get_variable(
                    name='b',
                    shape=[N_CLASS],
                    initializer=tf.constant_initializer(0.0))

cell = tf.contrib.rnn.BasicLSTMCell(STATE_SIZE)
cell = tf.contrib.rnn.DropoutWrapper(cell=cell,
                                     input_keep_prob=keep_prob,
                                     output_keep_prob=keep_prob)
sent_length = size(rnn_inputs)
outputs, final_state = tf.nn.dynamic_rnn(cell=cell,
                                         inputs=rnn_inputs,
                                         sequence_length=sent_length,
                                         dtype=tf.float32)

last = find_last(outputs, sent_length)
logits = tf.matmul(last, W_softmax) + b_softmax
# logits in shape [BATCH_SIZE, N_CLASS]
cross_entropy = tf.nn.softmax_cross_entropy_with_logits(logits=logits,
                                                        labels=y_)
# softmax cross entropy in shape [BATCH_SIZE, ]
# TODO mask padded loss to accelerate training
loss = tf.reduce_mean(cross_entropy)
train_step = tf.train.RMSPropOptimizer(1e-3).minimize(loss)

probs = tf.nn.softmax(logits)
correct = tf.equal(tf.argmax(probs, 1), tf.argmax(y_, 1))
accuracy = tf.reduce_mean(tf.cast(correct, tf.float32))

init = tf.global_variables_initializer()
init_embedd = tf.assign(embeddings, v, validate_shape=False)

sess = tf.Session()
sess.run(init)
sess.run(init_embedd, feed_dict={v: embedding_matrix})

with sess.as_default(), tf.device('/cpu:0'):
    data = flatten_split_resample(encoded_corpus, encoded_labels,
                                  valid_ratio=.2, sample_size=1000)
    train_sent_batch, train_label_batch = \
        batch_generator(*enqueue(*data[0]), BATCH_SIZE)
    valid_sent_batch, valid_label_batch = \
        batch_generator(*enqueue(*data[1]), BATCH_SIZE)
    train(300, x, y_, train_sent_batch, train_label_batch, valid_sent_batch,
          valid_label_batch, keep_prob, train_step, accuracy, loss, False)

def inference(query, sess, encoder=corpus_encoder, classes=classes, limit=5,
              decision_boundary=.85):
    """produce probabilities of most probable topic"""
    encoder, original_pad_length = encoder.fit([query]), encoder.pad_length
    encoded_query = encoder.encode(pad_length=original_pad_length)
    encoded_query = np.array(encoded_query).reshape(-1, original_pad_length)
    sess.run(fetches=init_embedd, feed_dict={v: encoder.vectorize()})
    probabilities = sess.run(fetches=probs,
                             feed_dict={x: encoded_query, keep_prob: 1})
    class_prob = np.mean(probabilities, axis=0).tolist()
    samples = [(class_, class_prob[k]) for k, class_ in enumerate(classes)]
    return feed_conversation(samples, limit, decision_boundary)
<commit_msg>join data pipeline to graph and remove data placeholders<commit_after>"""
    lstm (needs modularisation and further refactoring)

    sequence classification of variable lengths through
    Long-short Term Memory (LSTM) which models surrounding context of words.

    Words are represented in 300-dimension vectors to allow generalization over
    semantic redundancy.

    Distributed Representations of Words and Phrases and their Compositionality
    Mikolov et al. 2013
"""
import os
import random

import tensorflow as tf
import numpy as np

from tqdm import tqdm
from functools import wraps
from sklearn import model_selection, preprocessing

from chatbot.nlp.embedding import WordEmbedding
from chatbot.serializers import feed_conversation

from . import corpus, labels

os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2'

def resample(docs, labels, sample_size):
    unique_labels = np.unique(labels)
    indice_array = list()
    for label in unique_labels:
        indice_label = np.where(labels == label)[0].tolist()
        indice_array.extend(random.choices(indice_label, k=sample_size))
    return (docs[indice_array], labels[indice_array])

def flatten_split_resample(encoded_corpuses, encoded_labels,
                           valid_ratio=.2,
                           sample_size=1000):
    """break documents into sentences and augment, and one-hot encode labels"""

    flattened_docs = list()
    flattened_labels = list()
    for d, l in zip(encoded_corpuses, encoded_labels):
        for sent in d:
            flattened_docs.append(sent)
            flattened_labels.append(l)

    ravelled_corpus = np.array(flattened_docs)
    ravelled_labels = np.array(flattened_labels)

    X_train, X_test, y_train, y_test = \
        model_selection.train_test_split(ravelled_corpus, ravelled_labels,
                                         test_size=valid_ratio,
                                         stratify=ravelled_labels)

    return resample(X_train, y_train, sample_size),\
        resample(X_test, y_test, sample_size)

def size(sequence):
    """produces length vector in shape [batch_size, length] for each padded
    sequence of its true length excluding zero vectors.
    sequence of vectors comes in shape of [batch_size, STEP_SIZE, dimensions]
    """
    flag = tf.sign(tf.reduce_max(tf.abs(sequence), axis=2))
    length = tf.reduce_sum(flag, axis=1)
    return tf.cast(length, tf.int32)

def find_last(outputs, length):
    """zero padded sequences will produce zero vectors after sequence_length
    required using dynamic unrolling dynamic_rnn API. this function locates
    the last non-zero output for calculating logits
    """
    # outputs in shape [BATCH_SIZE, STEP_SIZE, STATE_SIZE], length [BATCH_SIZE]
    b, t, st = tf.unstack(tf.shape(outputs))
    flat_index = tf.range(b) * t + (length - 1)
    zeros = tf.zeros(shape=[b], dtype=tf.int32)
    non_neg_flat_index = tf.maximum(zeros, flat_index)
    flat_outputs = tf.reshape(outputs, [-1, st])
    last_outputs = tf.gather(flat_outputs, non_neg_flat_index)
    return last_outputs

def timeit(func):
    """calculate time for a function to complete"""
    import time

    def wrapper(*args, **kwargs):
        start = time.time()
        output = func(*args, **kwargs)
        end = time.time()
        print('function {0} took {1:0.3f} s'.format(
              func.__name__, (end - start) * 1))
        return output
    return wrapper

def multithreading(func):
    """decorator using tensorflow threading ability."""
    @wraps(func)
    def wrapper(*args, **kwargs):
        coord = tf.train.Coordinator()
        threads = tf.train.start_queue_runners(coord=coord)
        func_output = func(*args, **kwargs)
        try:
            coord.request_stop()
            coord.join(threads, stop_grace_period_secs=5)
        except (tf.errors.CancelledError, RuntimeError) as e:
            pass
        return func_output
    return wrapper

def enqueue(sent_encoded, label_encoded, num_epochs=None, shuffle=True):
    """returns an Ops Tensor with queued sentence and label pair"""
    sent = tf.convert_to_tensor(sent_encoded, dtype=tf.int32)
    label = tf.convert_to_tensor(label_encoded, dtype=tf.uint8)
    input_queue = tf.train.slice_input_producer(
                                tensor_list=[sent, label],
                                num_epochs=num_epochs,
                                shuffle=shuffle)
    return input_queue

def batch_generator(sent_queue, label_queue, batch_size=None, threads=4):
    return tf.train.shuffle_batch(
                    tensors=[sent_queue, label_queue],
                    batch_size=batch_size,
                    num_threads=threads,
                    capacity=1e3,
                    min_after_dequeue=batch_size,
                    allow_smaller_final_batch=True)

@multithreading
def train(n, is_train, optimiser, metric, loss, verbose):

    for global_step in tqdm(range(n), miniters=1, disable=verbose):
        _, train_accuracy, train_loss = \
            sess.run([optimiser, metric, loss], feed_dict={is_train: True})

        if verbose:
            print("step {0} of {3}, train accuracy: {1:.4f} log loss: {2:.4f}"
                  .format(global_step, train_accuracy, train_loss, n))

        if global_step and global_step % 100 == 0:
            valid_accuracy, valid_loss = sess.run(fetches=[metric, loss])

            print("step {0} of {3}, valid accuracy: {1:.4f} "
                  "log loss: {2:.4f}".format(global_step, valid_accuracy,
                                             valid_loss, n))

    print("steps {0} of {0}, train accuray: {1:.4f}, valid accuracy {2:.4f} "
          "log loss: {3:.4f}".format(n, valid_accuracy, train_accuracy,
                                     train_loss))

N_DOC = 20
corpus = corpus[:N_DOC]
labels = labels[:N_DOC]

STATE_SIZE = 24
STEP_SIZE = 30
N_CLASS = len(labels)
BATCH_SIZE = 50
EPOCH = 200

corpus_encoder = WordEmbedding(top=None).fit(corpus)
encoded_corpus = corpus_encoder.encode(zero_pad=True, pad_length=STEP_SIZE)

l_encoder = preprocessing.LabelBinarizer().fit(labels)
encoded_labels, classes = l_encoder.transform(labels), l_encoder.classes_

embedding_matrix = corpus_encoder.vectorize()
embed_shape = embedding_matrix.shape

data = flatten_split_resample(encoded_corpus, encoded_labels)
train_sent, train_label = batch_generator(*enqueue(*data[0]), BATCH_SIZE)
valid_sent, valid_label = batch_generator(*enqueue(*data[1]), BATCH_SIZE)

query = tf.placeholder_with_default(input=valid_sent,
                                    shape=[None, STEP_SIZE],
                                    name='query')
embeddings = tf.placeholder_with_default(input=embedding_matrix,
                                         shape=[None, embed_shape[1]],
                                         name='embeddings')
is_train = tf.placeholder_with_default(input=tf.constant(False), shape=[], name='is_train_or_valid')

feature_feed = tf.cond(is_train, lambda: train_sent, lambda: query)
label_feed = tf.cond(is_train, lambda: train_label, lambda: valid_label)
keep_prob = tf.cond(is_train, lambda: tf.constant(.5), lambda: tf.constant(1.))

rnn_inputs = tf.nn.embedding_lookup(embeddings, feature_feed)

with tf.variable_scope('softmax'):
    W_softmax = tf.get_variable(
                    name='W',
                    shape=[STATE_SIZE, N_CLASS],
                    initializer=tf.truncated_normal_initializer(stddev=0.1))
    b_softmax = tf.get_variable(
                    name='b',
                    shape=[N_CLASS],
                    initializer=tf.constant_initializer(0.0))

cell = tf.contrib.rnn.BasicLSTMCell(STATE_SIZE)
cell = tf.contrib.rnn.DropoutWrapper(cell=cell,
                                     input_keep_prob=keep_prob,
                                     output_keep_prob=keep_prob)
sent_length = size(rnn_inputs)
outputs, final_state = tf.nn.dynamic_rnn(cell=cell,
                                         inputs=rnn_inputs,
                                         sequence_length=sent_length,
                                         dtype=tf.float32)

last = find_last(outputs, sent_length)
logits = tf.matmul(last, W_softmax) + b_softmax
# logits in shape [BATCH_SIZE, N_CLASS]
cross_entropy = tf.nn.softmax_cross_entropy_with_logits(logits=logits,
                                                        labels=label_feed)
# softmax cross entropy in shape [BATCH_SIZE, ]
# TODO mask padded loss to accelerate training
loss = tf.reduce_mean(cross_entropy)
train_step = tf.train.RMSPropOptimizer(1e-3).minimize(loss)

probs = tf.nn.softmax(logits)
correct = tf.equal(tf.argmax(probs, 1), tf.argmax(label_feed, 1))
accuracy = tf.reduce_mean(tf.cast(correct, tf.float32))

init = tf.global_variables_initializer()

sess = tf.Session()
sess.run(init)
sess.graph.finalize()

with sess.as_default(), tf.device('/cpu:0'):
    train(5000, is_train, train_step, accuracy, loss, True)

def inference(question, sess, encoder=corpus_encoder, classes=classes,
              query=query, embeddings=embeddings, limit=5, decision_boundary=.85):
    """produce probabilities of most probable topic"""

    encoder, original_pad_length = encoder.fit([question]), encoder.pad_length
    encoded_query = encoder.encode(pad_length=original_pad_length)
    encoded_query = np.array(encoded_query).reshape(-1, original_pad_length)
    embedded_query = encoder.vectorize()
    probabilities = sess.run(fetches=probs,
                             feed_dict={query: encoded_query,
                                        embeddings: embedded_query})

    class_prob = np.mean(probabilities, axis=0).tolist()
    samples = [(class_, class_prob[k]) for k, class_ in enumerate(classes)]

    return feed_conversation(samples, limit, decision_boundary)
<|endoftext|>


<commit_before>import functools
import collections
import base64
import logging
import re

import numpy
from numpy.linalg import norm

import structure
from constants import KEYWORDS, NUMBERS
from mol_name import get_exact_name, autoflip_name, get_structure_type
from ml import get_decay_distance_correction_feature_vector, \
    get_naive_feature_vector, get_decay_feature_vector, \
    get_properties_from_decay_with_predictions
import dataparser

logger = logging.getLogger(__name__)

def cache(f):
    @functools.wraps(f)
    def wrapper(self, *args, **kwargs):
        name = '_' + f.__name__.lstrip('get')
        value = self.__dict__.get(name, None)
        if value is None:
            value = f(self, *args, **kwargs)
        self.__dict__[name] = value
        return self.__dict__[name]
    return wrapper

class Molecule(object):

    def __init__(self, name, **kwargs):
        self.name = name
        self.keywords = kwargs.get('keywords', [KEYWORDS])
        self.nprocshared = kwargs.get('nprocshared', 16)
        self.mem = kwargs.get('memory', 59)
        self.charge = kwargs.get('charge', 0)
        self.multiplicty = kwargs.get('multiplicity', 1)
        self.perturb = kwargs.get('perturb', 0.0)
        self.scale = kwargs.get('scale', 10)
        self._structure = None

    @property
    @cache
    def structure(self):
        return self._structure
    @structure.setter
    def structure(self, value):
        self._structure = value

    def from_gjf(self, f):
        self.structure = structure.from_gjf(f)

    def from_log(self, f):
        self.structure = structure.from_log(f)

    def get_gjf(self):
        starter = []
        if self.nprocshared is not None:
            starter.append("%%nprocshared=%d" % self.nprocshared)

        starter.extend([
            "%%mem=%dGB" % self.mem,
            "%%chk=%s.chk" % self.name,
            "# {keywords} {geom}",
            "",
            self.name,
            "",
            "%d %d" % (self.charge, self.multiplicty),
            ""
        ])
        header_template = "\n".join(starter)

        string = ''
        for i, keywords in enumerate(self.keywords):
            if not i:
                if self.structure.frozen:
                    geom = "geom=(modredundant,connectivity)"
                else:
                    geom = "geom=connectivity"
                string += header_template.format(keywords=keywords, geom=geom)
                string += self.structure.gjf
            else:
                string += "\n\n--Link1--\n"
                string += header_template.format(keywords=keywords, geom="geom=check guess=read")
        return string

    def get_mol2(self):
        header = "@<TRIPOS>MOLECULE\n%s\n" % self.name
        body = self.structure.mol2
        return header + body

    def get_png(self, scale=None):
        if scale is None:
            scale = self.scale
        return self.structure.draw(scale).getvalue()

    def get_png_data_url(self, scale=None):
        if scale is None:
            scale = self.scale
        string = "data:image/png;base64,"
        return string + base64.b64encode(self.get_png(scale))

    def get_svg(self, scale=None):
        if scale is None:
            scale = self.scale
        return self.structure.draw(scale, svg=True).getvalue()

    def get_svg_data_url(self, scale=None):
        if scale is None:
            scale = self.scale
        string = "data:image/svg;base64,"
        return string + base64.b64encode(self.get_svg(scale))

    @cache
    def get_coulomb_matrix(self):
        coords = []
        other = []

        for atom in self.structure.atoms:
            coords.append(atom.xyz)
            other.append(NUMBERS[atom.element])

        N = len(self.structure.atoms)
        data = numpy.matrix(numpy.zeros((N, N)))
        for i, x in enumerate(coords):
            for j, y in enumerate(coords[:i]):
                val = (other[i] * other[j]) / norm(x - y)
                data[i, j] = val
                data[j, i] = val

        diag = [0.5 * x ** 2.4 for x in other]
        for i, x in enumerate(diag):
            data[i, i] = x

        return data

    @cache
    def get_coulomb_matrix_feature(self):
        data = self.get_coulomb_matrix()
        vector = []
        end = []
        for i in xrange(data.shape[0]):
            for j in xrange(0, i):
                vector.append(data[i, j])
            end.append(data[i, i])
        return vector + end

    def get_element_counts(self):
        elems = [x.element for x in self.structure.atoms]
        return collections.Counter(elems)

    def get_formula(self):
        values = self.get_element_counts()
        return ''.join(key + str(values[key]) for key in sorted(values.keys()))

class Benzobisazole(Molecule):

    def __init__(self, name, **kwargs):
        super(Benzobisazole, self).__init__(name, **kwargs)
        if kwargs.get("autoflip"):
            self.name = autoflip_name(name)
        self._exact_name = None
        self._name_error = None

    @property
    @cache
    def structure(self):
        struct = structure.from_name(self.name)
        if self.perturb:
            struct.perturb(delta=self.perturb)
        return struct

    def get_exact_name(self, spacers=False):
        if self._exact_name is None:
            try:
                self._exact_name = get_exact_name(self.name, spacers=True)
            except Exception as e:
                self._name_error = str(e)
                self._exact_name = ''

        if spacers:
            return self._exact_name
        else:
            return self._exact_name.replace('*', '')

    @cache
    def get_structure_type(self):
        return get_structure_type(self.name)

    def get_name_error(self):
        self.get_exact_name()
        return self._name_error

    @cache
    def get_naive_feature_vector(self, **kwargs):
        try:
            exact_name = self.get_exact_name(spacers=True)
            return get_naive_feature_vector(exact_name, **kwargs)
        except ValueError:
            return None

    @cache
    def get_decay_feature_vector(self, **kwargs):
        try:
            exact_name = self.get_exact_name(spacers=True)
            return get_decay_feature_vector(exact_name, **kwargs)
        except ValueError:
            return None

    @cache
    def get_decay_distance_correction_feature_vector(self, **kwargs):
        try:
            exact_name = self.get_exact_name(spacers=True)
            return get_decay_distance_correction_feature_vector(exact_name,
                                                            **kwargs)
        except ValueError:
            return None

    @cache
    def get_property_predictions(self):
        try:
            feature = self.get_decay_feature_vector()
            results = get_properties_from_decay_with_predictions(feature)
        except ValueError:
            results = (None, None, None)
        return results

    @cache
    def get_property_limits(self):
        results = {
            "n": [None, None, None],
            "m": [None, None, None]
        }
        for direction in results:
            try:
                groups = []
                xvals = range(1, 5)
                for j in xvals:
                    if direction in self.name:
                        expr = "%s\d+" % direction
                        replace = "%s%d" % (direction, j)
                        temp_name = re.sub(expr, replace, self.name)
                    else:
                        temp_name = self.name + "_%s%d" % (direction, j)
                    struct = Benzobisazole(temp_name)
                    values = struct.get_property_predictions()
                    if values == (None, None, None):
                        raise ValueError
                    groups.append(values)

                lim_results = dataparser.predict_values(xvals, *zip(*groups))
                properties = ["homo", "lumo", "gap"]
                results[direction] = [lim_results[x][0] for x in properties]
            except ValueError:
                logger.info("Improper property limits: %s - %s" %
                            (self.name, direction))
                pass
        return results

    def get_info(self):
        features = [
            self.get_naive_feature_vector(),
            self.get_decay_feature_vector(),
        ]
        return {
            "molecule": self.name,
            "exact_name": self.get_exact_name(),
            "exact_name_spacers": self.get_exact_name(spacers=True),
            "features": features,
            "property_predictions": self.get_property_predictions(),
            "property_limits": self.get_property_limits(),
            "name_error": self.get_name_error(),
            "structure_type": self.get_structure_type(),
        }

<commit_msg>Fix issue where the keywords were just getting stored as strings<commit_after>import functools
import collections
import base64
import logging
import re

import numpy
from numpy.linalg import norm

import structure
from constants import KEYWORDS, NUMBERS
from mol_name import get_exact_name, autoflip_name, get_structure_type
from ml import get_decay_distance_correction_feature_vector, \
    get_naive_feature_vector, get_decay_feature_vector, \
    get_properties_from_decay_with_predictions
import dataparser

logger = logging.getLogger(__name__)

def cache(f):
    @functools.wraps(f)
    def wrapper(self, *args, **kwargs):
        name = '_' + f.__name__.lstrip('get')
        value = self.__dict__.get(name, None)
        if value is None:
            value = f(self, *args, **kwargs)
        self.__dict__[name] = value
        return self.__dict__[name]
    return wrapper

class Molecule(object):

    def __init__(self, name, **kwargs):
        self.name = name
        self.keywords = kwargs.get('keywords', KEYWORDS)
        if type(self.keywords) == str:
            self.keywords = [self.keywords]
        self.nprocshared = kwargs.get('nprocshared', 16)
        self.mem = kwargs.get('memory', 59)
        self.charge = kwargs.get('charge', 0)
        self.multiplicty = kwargs.get('multiplicity', 1)
        self.perturb = kwargs.get('perturb', 0.0)
        self.scale = kwargs.get('scale', 10)
        self._structure = None

    @property
    @cache
    def structure(self):
        return self._structure
    @structure.setter
    def structure(self, value):
        self._structure = value

    def from_gjf(self, f):
        self.structure = structure.from_gjf(f)

    def from_log(self, f):
        self.structure = structure.from_log(f)

    def get_gjf(self):
        starter = []
        if self.nprocshared is not None:
            starter.append("%%nprocshared=%d" % self.nprocshared)

        starter.extend([
            "%%mem=%dGB" % self.mem,
            "%%chk=%s.chk" % self.name,
            "# {keywords} {geom}",
            "",
            self.name,
            "",
            "%d %d" % (self.charge, self.multiplicty),
            ""
        ])
        header_template = "\n".join(starter)

        string = ''
        for i, keywords in enumerate(self.keywords):
            if not i:
                if self.structure.frozen:
                    geom = "geom=(modredundant,connectivity)"
                else:
                    geom = "geom=connectivity"
                string += header_template.format(keywords=keywords, geom=geom)
                string += self.structure.gjf
            else:
                string += "\n\n--Link1--\n"
                string += header_template.format(keywords=keywords, geom="geom=check guess=read")
        return string

    def get_mol2(self):
        header = "@<TRIPOS>MOLECULE\n%s\n" % self.name
        body = self.structure.mol2
        return header + body

    def get_png(self, scale=None):
        if scale is None:
            scale = self.scale
        return self.structure.draw(scale).getvalue()

    def get_png_data_url(self, scale=None):
        if scale is None:
            scale = self.scale
        string = "data:image/png;base64,"
        return string + base64.b64encode(self.get_png(scale))

    def get_svg(self, scale=None):
        if scale is None:
            scale = self.scale
        return self.structure.draw(scale, svg=True).getvalue()

    def get_svg_data_url(self, scale=None):
        if scale is None:
            scale = self.scale
        string = "data:image/svg;base64,"
        return string + base64.b64encode(self.get_svg(scale))

    @cache
    def get_coulomb_matrix(self):
        coords = []
        other = []

        for atom in self.structure.atoms:
            coords.append(atom.xyz)
            other.append(NUMBERS[atom.element])

        N = len(self.structure.atoms)
        data = numpy.matrix(numpy.zeros((N, N)))
        for i, x in enumerate(coords):
            for j, y in enumerate(coords[:i]):
                val = (other[i] * other[j]) / norm(x - y)
                data[i, j] = val
                data[j, i] = val

        diag = [0.5 * x ** 2.4 for x in other]
        for i, x in enumerate(diag):
            data[i, i] = x

        return data

    @cache
    def get_coulomb_matrix_feature(self):
        data = self.get_coulomb_matrix()
        vector = []
        end = []
        for i in xrange(data.shape[0]):
            for j in xrange(0, i):
                vector.append(data[i, j])
            end.append(data[i, i])
        return vector + end

    def get_element_counts(self):
        elems = [x.element for x in self.structure.atoms]
        return collections.Counter(elems)

    def get_formula(self):
        values = self.get_element_counts()
        return ''.join(key + str(values[key]) for key in sorted(values.keys()))

class Benzobisazole(Molecule):

    def __init__(self, name, **kwargs):
        super(Benzobisazole, self).__init__(name, **kwargs)
        if kwargs.get("autoflip"):
            self.name = autoflip_name(name)
        self._exact_name = None
        self._name_error = None

    @property
    @cache
    def structure(self):
        struct = structure.from_name(self.name)
        if self.perturb:
            struct.perturb(delta=self.perturb)
        return struct

    def get_exact_name(self, spacers=False):
        if self._exact_name is None:
            try:
                self._exact_name = get_exact_name(self.name, spacers=True)
            except Exception as e:
                self._name_error = str(e)
                self._exact_name = ''

        if spacers:
            return self._exact_name
        else:
            return self._exact_name.replace('*', '')

    @cache
    def get_structure_type(self):
        return get_structure_type(self.name)

    def get_name_error(self):
        self.get_exact_name()
        return self._name_error

    @cache
    def get_naive_feature_vector(self, **kwargs):
        try:
            exact_name = self.get_exact_name(spacers=True)
            return get_naive_feature_vector(exact_name, **kwargs)
        except ValueError:
            return None

    @cache
    def get_decay_feature_vector(self, **kwargs):
        try:
            exact_name = self.get_exact_name(spacers=True)
            return get_decay_feature_vector(exact_name, **kwargs)
        except ValueError:
            return None

    @cache
    def get_decay_distance_correction_feature_vector(self, **kwargs):
        try:
            exact_name = self.get_exact_name(spacers=True)
            return get_decay_distance_correction_feature_vector(exact_name,
                                                            **kwargs)
        except ValueError:
            return None

    @cache
    def get_property_predictions(self):
        try:
            feature = self.get_decay_feature_vector()
            results = get_properties_from_decay_with_predictions(feature)
        except ValueError:
            results = (None, None, None)
        return results

    @cache
    def get_property_limits(self):
        results = {
            "n": [None, None, None],
            "m": [None, None, None]
        }
        for direction in results:
            try:
                groups = []
                xvals = range(1, 5)
                for j in xvals:
                    if direction in self.name:
                        expr = "%s\d+" % direction
                        replace = "%s%d" % (direction, j)
                        temp_name = re.sub(expr, replace, self.name)
                    else:
                        temp_name = self.name + "_%s%d" % (direction, j)
                    struct = Benzobisazole(temp_name)
                    values = struct.get_property_predictions()
                    if values == (None, None, None):
                        raise ValueError
                    groups.append(values)

                lim_results = dataparser.predict_values(xvals, *zip(*groups))
                properties = ["homo", "lumo", "gap"]
                results[direction] = [lim_results[x][0] for x in properties]
            except ValueError:
                logger.info("Improper property limits: %s - %s" %
                            (self.name, direction))
                pass
        return results

    def get_info(self):
        features = [
            self.get_naive_feature_vector(),
            self.get_decay_feature_vector(),
        ]
        return {
            "molecule": self.name,
            "exact_name": self.get_exact_name(),
            "exact_name_spacers": self.get_exact_name(spacers=True),
            "features": features,
            "property_predictions": self.get_property_predictions(),
            "property_limits": self.get_property_limits(),
            "name_error": self.get_name_error(),
            "structure_type": self.get_structure_type(),
        }

<|endoftext|>


<commit_before>#!/usr/bin/env python
# coding: utf-8

DESCRIPTION="convert feature list into affinity matrix"

import numpy as np
import argparse
import sklearn.metrics as sm
import logging

import sys
from os.path import dirname
sys.path.append(dirname(__file__))
from my_target_counter import TargetCounter

logger = logging.getLogger(__file__)

def find_nth_smallest(a, n):
    return np.partition(a.flatten(), n-1)[n-1]

def calc_stsc_metric(X):
    D = sm.pairwise.pairwise_distances(X, metric='euclidean')
    sigma = [find_nth_smallest(ds,7) for ds in D]
    W = [[np.exp(-d*d/(sigma[i]*sigma[j])) for i,d in enumerate(ds)] for j,ds in enumerate(D)]
    return np.array(W)

def main(args):
    src_dir = args.src_dir
    dest_dir = args.dest_dir
    src_pat = "X_(\d{3}).csv$"
    tar_template = "W_%s.csv"
    tc=TargetCounter(src_pat,tar_template,src_dir,dest_dir)
    target_ids,src_files = tc.listup_targets()
    n_targets = len(target_ids)
    if args.count_targets:
        print(len(target_ids))
        sys.exit()
    if n_targets==0:
        logger.warn("There are no before-process src files in '%s'"%src_dir)
        sys.exit()

    for id,src_file in zip(target_ids,src_files):
        dest_file = "%s/%s"%(args.dest_dir,tc.id2destfile(id))
        #print(id,src_file,dest_file)
        X=np.loadtxt(src_file,delimiter=",")
        if args.metric=='euclidean':
            if args.gamma==None:
                W = calc_stsc_metric(X)
            else:
                W = sm.pairwise.rbf_kernel(X,gamma=args.gamma)
        elif args.metric=='cosine':
            W = sm.pairwise.cosine_similarity(X)
        else:
            logger.warn("unknown metric '%s'."%args.metric)
        #W = sm.pairwise.pairwise_distances(X, Y=None, metric=args.metric)
        np.savetxt(dest_file,W,delimiter=",")

parser = argparse.ArgumentParser(description=DESCRIPTION)
parser.add_argument('metric', \
        action='store', \
        nargs=None, \
        const=None, \
        default='euclidean', \
        type=str, \
        choices=None, \
        help='Metric for affinity calculation (‘cosine’, ‘euclidean’) (default: euclidean)', \
        metavar=None)

parser.add_argument('src_dir', \
        action='store', \
        nargs=None, \
        const=None, \
        default=None, \
        type=str, \
        choices=None, \
        help='Directory path where the source data are located.', \
        metavar=None)

parser.add_argument('dest_dir', \
        action='store', \
        nargs=None, \
        const=None, \
        default=None, \
        type=str, \
        choices=None, \
        help='Directory path where the dimension-reduced data will be located.', \
        metavar=None)

parser.add_argument('--gamma', \
        action='store', \
        nargs=None, \
        const=None, \
        default=None, \
        type=float, \
        choices=None, \
        help='gamma for rbf_kernel.', \
        metavar=None)

parser.add_argument('--count_targets',\
        action="store_true", default=False, help='count processing targets, and exit.')

if __name__ == '__main__':
    args = parser.parse_args()
    logger = logging.getLogger(__file__)
    logger.setLevel(logging.DEBUG)
    sh = logging.StreamHandler()
    logger.addHandler(sh)

    main(args)
<commit_msg>add scaling_unit option<commit_after>#!/usr/bin/env python
# coding: utf-8

DESCRIPTION="convert feature list into affinity matrix"

import numpy as np
import argparse
import sklearn.metrics as sm
import logging

import sys
from os.path import dirname
sys.path.append(dirname(__file__))
from my_target_counter import TargetCounter

logger = logging.getLogger(__file__)

def find_nth_smallest(a, n):
    return np.partition(a.flatten(), n-1)[n-1]

def calc_stsc_metric(X):
    D = sm.pairwise.pairwise_distances(X, metric='euclidean')
    sigma = [find_nth_smallest(ds,7) for ds in D]
    W = [[np.exp(-d*d/(sigma[i]*sigma[j])) for i,d in enumerate(ds)] for j,ds in enumerate(D)]
    return np.array(W)

def calc_data_driven_scale_unit(X,scale_unit):
    D = sm.pairwise.pairwise_distances(X, metric='euclidean')
    if scale_unit=='median':
        return np.median(D)
    else:
        return find_nth_smallest(np.ravel(D),int(scale_unit))

def main(args):
    src_dir = args.src_dir
    dest_dir = args.dest_dir
    src_pat = "X_(\d{3}).csv$"
    tar_template = "W_%s.csv"
    tc=TargetCounter(src_pat,tar_template,src_dir,dest_dir)
    target_ids,src_files = tc.listup_targets()
    n_targets = len(target_ids)
    if args.count_targets:
        print(len(target_ids))
        sys.exit()
    if n_targets==0:
        logger.warn("There are no before-process src files in '%s'"%src_dir)
        sys.exit()

    for id,src_file in zip(target_ids,src_files):
        dest_file = "%s/%s"%(args.dest_dir,tc.id2destfile(id))
        #print(id,src_file,dest_file)
        X=np.loadtxt(src_file,delimiter=",")
        if args.metric=='euclidean':
            if args.gamma==None:
                W = calc_stsc_metric(X)
            else:

                W = sm.pairwise.rbf_kernel(X,gamma=args.gamma)
        elif args.metric=='cosine':
            W = sm.pairwise.cosine_similarity(X)
        else:
            logger.warn("unknown metric '%s'."%args.metric)
        #W = sm.pairwise.pairwise_distances(X, Y=None, metric=args.metric)
        np.savetxt(dest_file,W,delimiter=",")

parser = argparse.ArgumentParser(description=DESCRIPTION)
parser.add_argument('metric', \
        action='store', \
        nargs=None, \
        const=None, \
        default='euclidean', \
        type=str, \
        choices=None, \
        help='Metric for affinity calculation (‘cosine’, ‘euclidean’) (default: euclidean)', \
        metavar=None)

parser.add_argument('src_dir', \
        action='store', \
        nargs=None, \
        const=None, \
        default=None, \
        type=str, \
        choices=None, \
        help='Directory path where the source data are located.', \
        metavar=None)

parser.add_argument('dest_dir', \
        action='store', \
        nargs=None, \
        const=None, \
        default=None, \
        type=str, \
        choices=None, \
        help='Directory path where the dimension-reduced data will be located.', \
        metavar=None)

parser.add_argument('--gamma', \
        action='store', \
        nargs=None, \
        const=None, \
        default=1.0, \
        type=float, \
        choices=None, \
        help='gamma for rbf_kernel.', \
        metavar=None)

parser.add_argument('-s','--scale_unit', \
        action='store', \
        nargs=None, \
        const=None, \
        default='median', \
        type=str, \
        choices=None, \
        help='Strategy to obtain a data-driven scale unit lamda (gamma*lamda is used as sigma rbf kernel). "median" or an int value (to use the n-th largest distance). (default:median)', \
        metavar=None)

parser.add_argument('--count_targets',\
        action="store_true", default=False, help='count processing targets, and exit.')

if __name__ == '__main__':
    args = parser.parse_args()
    logger = logging.getLogger(__file__)
    logger.setLevel(logging.DEBUG)
    sh = logging.StreamHandler()
    logger.addHandler(sh)

    main(args)
<|endoftext|>