<commit_before>#!/usr/bin/python
"""This script compiles multiple instances of a program trying out different
heuristics, and storing in the database the best one that is found"""
import sys
import os
import shutil
import sqlite3
import random
import xml.dom.minidom
import re
import pbutil
import tunerwarnings
import maximaparser
from candidatetester import Candidate
from xml.sax.saxutils import escape
from sgatuner import autotune
from tunerconfig import config

#--------- Config ------------------
conf_deleteTempDir = True
conf_minTrialNumber = 10
conf_probabilityExploration = 0.7
conf_pickBestN = 3
#--------- Autotuner config --------
config.max_time=30 #Seconds
#-----------------------------------

class HeuristicDB:
  def __init__(self):
    #Open DB
    self.__db = sqlite3.connect(self.computeDBPath())
    self.__createTables()
    self.__bestNCache= dict()

  def __createTable(self, name, params):
    cur = self.__db.cursor()
    query = "CREATE TABLE IF NOT EXISTS '"+name+"' "+params
    cur.execute(query)
    cur.close()
    self.__db.commit()

  def __createTables(self):
    self.__createTable("HeuristicKind", "('ID' INTEGER PRIMARY KEY AUTOINCREMENT, "
                                        "'name' TEXT UNIQUE)")
    self.__createTable("Heuristic", "('kindID' INTEGER, 'formula' TEXT, "
                                    "'useCount' INTEGER, 'score' FLOAT,"
                                    "PRIMARY KEY (kindID, formula), "
                                    "FOREIGN KEY ('kindID') REFERENCES 'HeuristicKind' ('ID')"
                                    "ON DELETE CASCADE ON UPDATE CASCADE)")
    #TODO:self.__createTable("InSet", "('setID' INTEGER, 'heuristicID' INTEGER)"

  def computeDBPath(self):
    #TODO: make the path more flexible
    dbPath= os.path.expanduser(config.output_dir+"/knowledge.db")
    return dbPath

  def getHeuristicKindID(self, kindName):
    cur = self.__db.cursor()
    query = "SELECT ID From HeuristicKind WHERE name='"+kindName+"'"
    cur.execute(query)
    kindID = cur.fetchone()[0]
    cur.close()
    return kindID

  def storeHeuristicKind(self, kindName):
    cur = self.__db.cursor()
    query = "INSERT OR IGNORE INTO HeuristicKind ('name') VALUES ('"+kindName+"')"
    cur.execute(query)
    cur.close()
    self.__db.commit()
    return self.getHeuristicKindID(kindName)

  def increaseHeuristicScore(self, name, formula, score):
    print "INCREASING by " + str(score) + ": " + formula
    kindID=self.storeHeuristicKind(name)
    cur = self.__db.cursor()
    query = "UPDATE Heuristic SET score=score+? WHERE kindID=? AND formula=?"
    cur.execute(query, (score, kindID, formula))
    if cur.rowcount == 0:
      #There was no such heuristic in the DB: probably it was taken from the defaults
      query = "INSERT INTO Heuristic (kindID, formula, useCount, score) VALUES (?, ?, 1, ?)"
      cur.execute(query, (kindID, formula, score))
    cur.close()
    self.__db.commit()

  def increaseHeuristicUseCount(self, name, formula):
    kindID=self.storeHeuristicKind(name)
    cur = self.__db.cursor()
    query = "UPDATE Heuristic SET useCount=useCount+1 WHERE kindID=? AND formula=?"
    cur.execute(query, (kindID, formula))
    if cur.rowcount == 0:
      #There was no such heuristic in the DB: let's add it
      query = "INSERT INTO Heuristic (kindID, formula, useCount, score) VALUES (?, ?, 1, 0)"
      cur.execute(query, (kindID, formula))
    cur.close()
    self.__db.commit()

  def increaseScore(self, hSet, score):
    """Mark a set of heuristics as selected as the best one for an executable"""
    #TODO: also store it as a set
    for name, formula in hSet.iteritems():
      self.increaseHeuristicScore(name, formula, score)

  def markAsUsed(self, hSet):
    """Mark a set of heuristics as used for generating a candidate executable"""
    #TODO: also store it as a set
    for name, formula in hSet.iteritems():
      print "USED: " + formula
      self.increaseHeuristicUseCount(name, formula)

  def getBestNHeuristics(self, name, N):
    try:
      cached = self.__bestNCache[name]
      return cached
    except:
      #Not in the cache
      #Fall back to accessing the db
      pass
    cur = self.__db.cursor()
    query = "SELECT formula FROM Heuristic JOIN HeuristicKind ON Heuristic.kindID=HeuristicKind.ID WHERE HeuristicKind.name=? ORDER BY Heuristic.score/Heuristic.useCount DESC LIMIT ?"
    cur.execute(query, (name, N))
    result = [row[0] for row in cur.fetchall()]
    cur.close()
    self.__bestNCache[name]=result
    return result

class HeuristicSet(dict):
  def toXmlStrings(self):
    return ["<heuristic name=\""+name+"\" formula=\""+escape(self[name])+"\" />" for name in self]

  def toXmlFile(self, filename):
    outfile = open(filename, "w")
    outfile.write("<heuristics>\n")
    for xmlstring in self.toXmlStrings():
      outfile.write("\t")
      outfile.write(xmlstring)
      outfile.write("\n")
    outfile.write("</heuristics>\n")
    outfile.close()

  def importFromXml(self, xmlDOM):
    for heuristicXML in xmlDOM.getElementsByTagName("heuristic"):
      name=heuristicXML.getAttribute("name")
      formula=heuristicXML.getAttribute("formula")
      self[name] = formula

  def complete(self, heuristicNames, db, N):
    """Complete the sets using the given db, so that it contains all the
heuristics specified in the heuristicNames list.

Every missing heuristic is completed with one randomly taken from the best N
heuristics in the database  """
    random.seed()
    #Find the missing heuristics
    missingHeuristics = list(heuristicNames)
    for name in self:
      try:
        missingHeuristics.remove(name)
      except ValueError:
        #A heuristic could be in the input file, but useless, therefore not in
        #the missingHeuristic list
        pass

    #Complete the set
    for heuristic in missingHeuristics:
      bestN=db.getBestNHeuristics(heuristic, N)
      if len(bestN) == 0:
        #No such heuristic in the DB. Do not complete the set
        #This is not a problem. It's probably a new heuristic:
        #just ignore it and it will fall back on the default implemented
        #into the compiler
        continue
      formula=random.choice(bestN)

      if random.random() < conf_probabilityExploration:
        #Generete a new formula by modifying the existing one
        formulaObj = maximaparser.parse(formula)
        formulaObj.evolveValue()
        formula = str(formulaObj)

      self[heuristic] = formula

class HeuristicManager:
  """Manages sets of heuristics stored in a file with the following format:
<heuristics>
  <set>
    <heuristic name="heuristicName" formula="a+b+c" />
    <heuristic name="heuristicName2" formula="a+b+d" />
  </set>
  <set>
    <heuristic name="heuristicName3" formula="x+y*z" />
    <heuristic name="heuristicName4" formula="a+g+s" />
  </set>
</heuristics>
"""
  def __init__(self, heuristicSetFileName=None):
    self.__heuristicSets = []
    if heuristicSetFileName is not None:
      self.__xml = xml.dom.minidom.parse(heuristicSetFileName)

      # Extract information
      for hSet in self.__xml.getElementsByTagName("set"):
        self.__heuristicSets.append(self.__parseHeuristicSet(hSet))

  def __parseHeuristicSet(self, hSetXML):
    """Parses a xml heuristic set returning it as a list of pairs name-formula"""
    hSet = HeuristicSet()
    hSet.importFromXml(hSetXML)
    return hSet

  def heuristicSet(self, i):
    """Get the i-th heuristic set"""
    return self.__heuristicSets[i]

  def allHeuristicSets(self):
    return self.__heuristicSets

def candidateKey(candidate):
  """Generates a comparison key for a candidate.
Candidates are sorted by the number of dimensions (the highest, the better),
then by average execution time of the biggest dimension (the lower the better)"""
  if candidate is None:
    return (float('inf'), float('inf'))
  numDimensions = len(candidate.metrics[0])
  executionTime = candidate.metrics[0][2**(numDimensions-1)].mean()
  return (1/numDimensions, executionTime)

class CandidateList(list):
  def addOriginalIndex(self):
    count = 0
    for candidate in self:
      if candidate is None:
        continue
      candidate.originalIndex = count;
      count = count + 1

  def sortBySpeed(self):
    """Adds the "score" and "originalIndex" attributes to every candidate.
    Also, sorts the list by score"""
    self.sort(key=candidateKey)

class LearningCompiler:
  def __init__(self, pbcExe, heuristicSetFileName=None, jobs=None):
    self.__heuristicManager = HeuristicManager(heuristicSetFileName)
    self.__minTrialNumber = conf_minTrialNumber
    self.__pbcExe = pbcExe
    self.__jobs=jobs
    self.__db = HeuristicDB()

  def storeCandidatesDataInDB(self, candidates, basesubdir, basename):
    """Store data from all the info file, with score.
The candidates should already be ordered (from the best to the worst) and
with the originalIndex field added"""
    numCandidates = len(candidates)
    count=0
    for candidate in candidates:
      infoFile=os.path.join(basesubdir,
                            str(candidate.originalIndex),
                            basename+".info")

      score = (numCandidates - count) / float(numCandidates)

       #Take the data about the used heuristics scores and store it into the DB
      infoxml = xml.dom.minidom.parse(infoFile)
      hSet = HeuristicSet()
      hSet.importFromXml(infoxml)
      self.__db.increaseScore(hSet, score)
      self.__db.markAsUsed(hSet)

      count = count +1

  def compileLearningHeuristics(self, benchmark, finalBinary=None):
    #Define file names
    path, basenameExt = os.path.split(benchmark)
    if path == "":
      path="./"
    basename, ext = os.path.splitext(basenameExt)
    basesubdir=os.path.join(path,basename+".tmp")
    #Init variables
    candidates=CandidateList()

    #Compile with current best heuristics
    outDir = os.path.join(basesubdir, "0")
    if not os.path.isdir(outDir):
      #Create the output directory
      os.makedirs(outDir)
    binary= os.path.join(outDir, basename)
    status=pbutil.compileBenchmark(self.__pbcExe, benchmark, binary=binary, jobs=self.__jobs)
    if status != 0:
      return status

    try:
      autotune(binary, candidates)
    except tunerwarnings.AlwaysCrashes:
        print "Current best Candidate always crashes!"
        #Add an empty entry for the candidate
        candidates.append(None)

    #Get the full set of heuristics used
    infoFile=binary+".info"
    currentBestHSet = HeuristicSet()
    currentBestHSet.importFromXml(xml.dom.minidom.parse(infoFile))
    neededHeuristics = currentBestHSet.keys()

    #Get hSets
    allHSets = self.__heuristicManager.allHeuristicSets()
    while len(allHSets) < (self.__minTrialNumber): #Not enough hSets!
      allHSets.append(HeuristicSet())

    numSets = len(allHSets)

    count=1
    for hSet in allHSets:
      hSet.complete(neededHeuristics, self.__db, conf_pickBestN)

      #Define more file names
      outDir = os.path.join(basesubdir, str(count))
      if not os.path.isdir(outDir):
        #Create the output directory
        os.makedirs(outDir)
      binary= os.path.join(outDir, basename)

      heuristicsFile= os.path.join(outDir, "heuristics.txt")
      hSet.toXmlFile(heuristicsFile)

      status = pbutil.compileBenchmark(self.__pbcExe, benchmark, binary=binary, heuristics=heuristicsFile, jobs=self.__jobs)
      if status != 0:
        print "Compile FAILED"
        print "while using heuristics: "
        print hSet
        return status

      #Autotune
      try:
        autotune(binary, candidates)
      except tunerwarnings.AlwaysCrashes:
        print "Candidate "+str(count)+" always crashes!"
        #Add an empty entry for the candidate
        candidates.append(None)

      count = count + 1

    candidates.addOriginalIndex()
    candidates.sortBySpeed()

    if candidates[0] is None:
      raise tunerwarnings.AlwaysCrashes()

    self.storeCandidatesDataInDB(candidates, basesubdir, basename)

    bestIndex = candidates[0].originalIndex

    print "The best candidate is: "+str(bestIndex)

    #Move every file to the right place
    bestSubDir=os.path.join(basesubdir, str(bestIndex))
    #  compiled program:
    bestBin=os.path.join(bestSubDir, basename)
    if finalBinary is not None:
      finalBin=finalBinary
    else:
      finalBin=os.path.join(path, basename)
    shutil.move(bestBin, finalBin)
    #  .cfg file
    bestCfg=os.path.join(bestSubDir, basename+".cfg")
    finalCfg=finalBin + ".cfg"
    shutil.move(bestCfg, finalCfg)
    #  .info file
    bestInfo=os.path.join(bestSubDir, basename+".info")
    finalInfo=finalBin+".info"
    shutil.move(bestInfo, finalInfo)
    #  .obj directory
    bestObjDir=os.path.join(bestSubDir, basename+".obj")
    destObjDir=finalBin+".obj"
    if os.path.isdir(destObjDir):
      shutil.rmtree(destObjDir)
    shutil.move(bestObjDir, destObjDir)
    #  input heuristic file
    if bestIndex != 0: #Program 0 is run with only the best heuristics in the DB
      bestHeurFile=os.path.join(bestSubDir, "heuristics.txt")
      finalHeurFile=finalBin+".heur"
      shutil.move(bestHeurFile, finalHeurFile)

    #Delete all the rest
    if conf_deleteTempDir:
      shutil.rmtree(basesubdir)

    return 0

#TEST
if __name__ == "__main__":
  #basedir="/afs/csail.mit.edu/u/m/mtartara/programs/petabricks/"
  basedir="/home/mikyt/programmi/petabricks/"
  pbc=basedir+"src/pbc"
  l=LearningCompiler(pbc, sys.argv[1], conf_minTrialNumber)
  l.compileLearningHeuristics(sys.argv[2])
<commit_msg>LOG prints removed<commit_after>#!/usr/bin/python
"""This script compiles multiple instances of a program trying out different
heuristics, and storing in the database the best one that is found"""
import sys
import os
import shutil
import sqlite3
import random
import xml.dom.minidom
import re
import pbutil
import tunerwarnings
import maximaparser
from candidatetester import Candidate
from xml.sax.saxutils import escape
from sgatuner import autotune
from tunerconfig import config

#--------- Config ------------------
conf_deleteTempDir = True
conf_minTrialNumber = 10
conf_probabilityExploration = 0.7
conf_pickBestN = 3
#--------- Autotuner config --------
config.max_time=30 #Seconds
#-----------------------------------

class HeuristicDB:
  def __init__(self):
    #Open DB
    self.__db = sqlite3.connect(self.computeDBPath())
    self.__createTables()
    self.__bestNCache= dict()

  def __createTable(self, name, params):
    cur = self.__db.cursor()
    query = "CREATE TABLE IF NOT EXISTS '"+name+"' "+params
    cur.execute(query)
    cur.close()
    self.__db.commit()

  def __createTables(self):
    self.__createTable("HeuristicKind", "('ID' INTEGER PRIMARY KEY AUTOINCREMENT, "
                                        "'name' TEXT UNIQUE)")
    self.__createTable("Heuristic", "('kindID' INTEGER, 'formula' TEXT, "
                                    "'useCount' INTEGER, 'score' FLOAT,"
                                    "PRIMARY KEY (kindID, formula), "
                                    "FOREIGN KEY ('kindID') REFERENCES 'HeuristicKind' ('ID')"
                                    "ON DELETE CASCADE ON UPDATE CASCADE)")
    #TODO:self.__createTable("InSet", "('setID' INTEGER, 'heuristicID' INTEGER)"

  def computeDBPath(self):
    #TODO: make the path more flexible
    dbPath= os.path.expanduser(config.output_dir+"/knowledge.db")
    return dbPath

  def getHeuristicKindID(self, kindName):
    cur = self.__db.cursor()
    query = "SELECT ID From HeuristicKind WHERE name='"+kindName+"'"
    cur.execute(query)
    kindID = cur.fetchone()[0]
    cur.close()
    return kindID

  def storeHeuristicKind(self, kindName):
    cur = self.__db.cursor()
    query = "INSERT OR IGNORE INTO HeuristicKind ('name') VALUES ('"+kindName+"')"
    cur.execute(query)
    cur.close()
    self.__db.commit()
    return self.getHeuristicKindID(kindName)

  def increaseHeuristicScore(self, name, formula, score):
    kindID=self.storeHeuristicKind(name)
    cur = self.__db.cursor()
    query = "UPDATE Heuristic SET score=score+? WHERE kindID=? AND formula=?"
    cur.execute(query, (score, kindID, formula))
    if cur.rowcount == 0:
      #There was no such heuristic in the DB: probably it was taken from the defaults
      query = "INSERT INTO Heuristic (kindID, formula, useCount, score) VALUES (?, ?, 1, ?)"
      cur.execute(query, (kindID, formula, score))
    cur.close()
    self.__db.commit()

  def increaseHeuristicUseCount(self, name, formula):
    kindID=self.storeHeuristicKind(name)
    cur = self.__db.cursor()
    query = "UPDATE Heuristic SET useCount=useCount+1 WHERE kindID=? AND formula=?"
    cur.execute(query, (kindID, formula))
    if cur.rowcount == 0:
      #There was no such heuristic in the DB: let's add it
      query = "INSERT INTO Heuristic (kindID, formula, useCount, score) VALUES (?, ?, 1, 0)"
      cur.execute(query, (kindID, formula))
    cur.close()
    self.__db.commit()

  def increaseScore(self, hSet, score):
    """Mark a set of heuristics as selected as the best one for an executable"""
    #TODO: also store it as a set
    for name, formula in hSet.iteritems():
      self.increaseHeuristicScore(name, formula, score)

  def markAsUsed(self, hSet):
    """Mark a set of heuristics as used for generating a candidate executable"""
    #TODO: also store it as a set
    for name, formula in hSet.iteritems():
      self.increaseHeuristicUseCount(name, formula)

  def getBestNHeuristics(self, name, N):
    try:
      cached = self.__bestNCache[name]
      return cached
    except:
      #Not in the cache
      #Fall back to accessing the db
      pass
    cur = self.__db.cursor()
    query = "SELECT formula FROM Heuristic JOIN HeuristicKind ON Heuristic.kindID=HeuristicKind.ID WHERE HeuristicKind.name=? ORDER BY Heuristic.score/Heuristic.useCount DESC LIMIT ?"
    cur.execute(query, (name, N))
    result = [row[0] for row in cur.fetchall()]
    cur.close()
    self.__bestNCache[name]=result
    return result

class HeuristicSet(dict):
  def toXmlStrings(self):
    return ["<heuristic name=\""+name+"\" formula=\""+escape(self[name])+"\" />" for name in self]

  def toXmlFile(self, filename):
    outfile = open(filename, "w")
    outfile.write("<heuristics>\n")
    for xmlstring in self.toXmlStrings():
      outfile.write("\t")
      outfile.write(xmlstring)
      outfile.write("\n")
    outfile.write("</heuristics>\n")
    outfile.close()

  def importFromXml(self, xmlDOM):
    for heuristicXML in xmlDOM.getElementsByTagName("heuristic"):
      name=heuristicXML.getAttribute("name")
      formula=heuristicXML.getAttribute("formula")
      self[name] = formula

  def complete(self, heuristicNames, db, N):
    """Complete the sets using the given db, so that it contains all the
heuristics specified in the heuristicNames list.

Every missing heuristic is completed with one randomly taken from the best N
heuristics in the database  """
    random.seed()
    #Find the missing heuristics
    missingHeuristics = list(heuristicNames)
    for name in self:
      try:
        missingHeuristics.remove(name)
      except ValueError:
        #A heuristic could be in the input file, but useless, therefore not in
        #the missingHeuristic list
        pass

    #Complete the set
    for heuristic in missingHeuristics:
      bestN=db.getBestNHeuristics(heuristic, N)
      if len(bestN) == 0:
        #No such heuristic in the DB. Do not complete the set
        #This is not a problem. It's probably a new heuristic:
        #just ignore it and it will fall back on the default implemented
        #into the compiler
        continue
      formula=random.choice(bestN)

      if random.random() < conf_probabilityExploration:
        #Generete a new formula by modifying the existing one
        formulaObj = maximaparser.parse(formula)
        formulaObj.evolveValue()
        formula = str(formulaObj)

      self[heuristic] = formula

class HeuristicManager:
  """Manages sets of heuristics stored in a file with the following format:
<heuristics>
  <set>
    <heuristic name="heuristicName" formula="a+b+c" />
    <heuristic name="heuristicName2" formula="a+b+d" />
  </set>
  <set>
    <heuristic name="heuristicName3" formula="x+y*z" />
    <heuristic name="heuristicName4" formula="a+g+s" />
  </set>
</heuristics>
"""
  def __init__(self, heuristicSetFileName=None):
    self.__heuristicSets = []
    if heuristicSetFileName is not None:
      self.__xml = xml.dom.minidom.parse(heuristicSetFileName)

      # Extract information
      for hSet in self.__xml.getElementsByTagName("set"):
        self.__heuristicSets.append(self.__parseHeuristicSet(hSet))

  def __parseHeuristicSet(self, hSetXML):
    """Parses a xml heuristic set returning it as a list of pairs name-formula"""
    hSet = HeuristicSet()
    hSet.importFromXml(hSetXML)
    return hSet

  def heuristicSet(self, i):
    """Get the i-th heuristic set"""
    return self.__heuristicSets[i]

  def allHeuristicSets(self):
    return self.__heuristicSets

def candidateKey(candidate):
  """Generates a comparison key for a candidate.
Candidates are sorted by the number of dimensions (the highest, the better),
then by average execution time of the biggest dimension (the lower the better)"""
  if candidate is None:
    return (float('inf'), float('inf'))
  numDimensions = len(candidate.metrics[0])
  executionTime = candidate.metrics[0][2**(numDimensions-1)].mean()
  return (1/numDimensions, executionTime)

class CandidateList(list):
  def addOriginalIndex(self):
    count = 0
    for candidate in self:
      if candidate is None:
        continue
      candidate.originalIndex = count;
      count = count + 1

  def sortBySpeed(self):
    """Adds the "score" and "originalIndex" attributes to every candidate.
    Also, sorts the list by score"""
    self.sort(key=candidateKey)

class LearningCompiler:
  def __init__(self, pbcExe, heuristicSetFileName=None, jobs=None):
    self.__heuristicManager = HeuristicManager(heuristicSetFileName)
    self.__minTrialNumber = conf_minTrialNumber
    self.__pbcExe = pbcExe
    self.__jobs=jobs
    self.__db = HeuristicDB()

  def storeCandidatesDataInDB(self, candidates, basesubdir, basename):
    """Store data from all the info file, with score.
The candidates should already be ordered (from the best to the worst) and
with the originalIndex field added"""
    numCandidates = len(candidates)
    count=0
    for candidate in candidates:
      infoFile=os.path.join(basesubdir,
                            str(candidate.originalIndex),
                            basename+".info")

      score = (numCandidates - count) / float(numCandidates)

       #Take the data about the used heuristics scores and store it into the DB
      infoxml = xml.dom.minidom.parse(infoFile)
      hSet = HeuristicSet()
      hSet.importFromXml(infoxml)
      self.__db.increaseScore(hSet, score)
      self.__db.markAsUsed(hSet)

      count = count +1

  def compileLearningHeuristics(self, benchmark, finalBinary=None):
    #Define file names
    path, basenameExt = os.path.split(benchmark)
    if path == "":
      path="./"
    basename, ext = os.path.splitext(basenameExt)
    basesubdir=os.path.join(path,basename+".tmp")
    #Init variables
    candidates=CandidateList()

    #Compile with current best heuristics
    outDir = os.path.join(basesubdir, "0")
    if not os.path.isdir(outDir):
      #Create the output directory
      os.makedirs(outDir)
    binary= os.path.join(outDir, basename)
    status=pbutil.compileBenchmark(self.__pbcExe, benchmark, binary=binary, jobs=self.__jobs)
    if status != 0:
      return status

    try:
      autotune(binary, candidates)
    except tunerwarnings.AlwaysCrashes:
        print "Current best Candidate always crashes!"
        #Add an empty entry for the candidate
        candidates.append(None)

    #Get the full set of heuristics used
    infoFile=binary+".info"
    currentBestHSet = HeuristicSet()
    currentBestHSet.importFromXml(xml.dom.minidom.parse(infoFile))
    neededHeuristics = currentBestHSet.keys()

    #Get hSets
    allHSets = self.__heuristicManager.allHeuristicSets()
    while len(allHSets) < (self.__minTrialNumber): #Not enough hSets!
      allHSets.append(HeuristicSet())

    numSets = len(allHSets)

    count=1
    for hSet in allHSets:
      hSet.complete(neededHeuristics, self.__db, conf_pickBestN)

      #Define more file names
      outDir = os.path.join(basesubdir, str(count))
      if not os.path.isdir(outDir):
        #Create the output directory
        os.makedirs(outDir)
      binary= os.path.join(outDir, basename)

      heuristicsFile= os.path.join(outDir, "heuristics.txt")
      hSet.toXmlFile(heuristicsFile)

      status = pbutil.compileBenchmark(self.__pbcExe, benchmark, binary=binary, heuristics=heuristicsFile, jobs=self.__jobs)
      if status != 0:
        print "Compile FAILED"
        print "while using heuristics: "
        print hSet
        return status

      #Autotune
      try:
        autotune(binary, candidates)
      except tunerwarnings.AlwaysCrashes:
        print "Candidate "+str(count)+" always crashes!"
        #Add an empty entry for the candidate
        candidates.append(None)

      count = count + 1

    candidates.addOriginalIndex()
    candidates.sortBySpeed()

    if candidates[0] is None:
      raise tunerwarnings.AlwaysCrashes()

    self.storeCandidatesDataInDB(candidates, basesubdir, basename)

    bestIndex = candidates[0].originalIndex

    print "The best candidate is: "+str(bestIndex)

    #Move every file to the right place
    bestSubDir=os.path.join(basesubdir, str(bestIndex))
    #  compiled program:
    bestBin=os.path.join(bestSubDir, basename)
    if finalBinary is not None:
      finalBin=finalBinary
    else:
      finalBin=os.path.join(path, basename)
    shutil.move(bestBin, finalBin)
    #  .cfg file
    bestCfg=os.path.join(bestSubDir, basename+".cfg")
    finalCfg=finalBin + ".cfg"
    shutil.move(bestCfg, finalCfg)
    #  .info file
    bestInfo=os.path.join(bestSubDir, basename+".info")
    finalInfo=finalBin+".info"
    shutil.move(bestInfo, finalInfo)
    #  .obj directory
    bestObjDir=os.path.join(bestSubDir, basename+".obj")
    destObjDir=finalBin+".obj"
    if os.path.isdir(destObjDir):
      shutil.rmtree(destObjDir)
    shutil.move(bestObjDir, destObjDir)
    #  input heuristic file
    if bestIndex != 0: #Program 0 is run with only the best heuristics in the DB
      bestHeurFile=os.path.join(bestSubDir, "heuristics.txt")
      finalHeurFile=finalBin+".heur"
      shutil.move(bestHeurFile, finalHeurFile)

    #Delete all the rest
    if conf_deleteTempDir:
      shutil.rmtree(basesubdir)

    return 0

#TEST
if __name__ == "__main__":
  #basedir="/afs/csail.mit.edu/u/m/mtartara/programs/petabricks/"
  basedir="/home/mikyt/programmi/petabricks/"
  pbc=basedir+"src/pbc"
  l=LearningCompiler(pbc, sys.argv[1], conf_minTrialNumber)
  l.compileLearningHeuristics(sys.argv[2])
<|endoftext|>


<commit_before>#!/usr/bin/env conda-execute

# conda execute
# env:
#  - python
#  - conda-smithy >=1.1
#  - gitpython
#  - pygithub
# channels:
#  - conda-forge
# run_with: python

import argparse
from contextlib import contextmanager
import os
import random
import re
import textwrap
import time

import git
import github

import conda_smithy.github
import conda_smithy.feedstocks as feedstocks

# The version commented right after the comma is the one available for **Linux** in **defaults**.
# That version is assumed to be the same in all platforms and the one used to build all packages.
# BE VERY CAREFUL when updating pinnings. Make sure to check https://abi-laboratory.pro/tracker/
# for any incompatibility between versions.
# Also be aware that currently the following pinnings are used for both build and run meta.yaml
# sections. Defining the same pinning for both sections means we need to be more restrictive.
#
# For example, we used to define gmp as 'gmp >=5.0.1,<7'. This is wrong as it allows a package
# to be built against gmp 6.1.0 but run with 5.0.1.
# As showed by https://abi-laboratory.pro/tracker/timeline/gmp/ binaries built with 6.1.0 are not
# compatible with 5.0.1 (the other way around would be true).
pinned = {
          'boost': 'boost 1.64.*',  # 1.61.0
          'boost-cpp': 'boost-cpp 1.64.*',  # NA
          'bzip2': 'bzip2 1.0.*',  # 1.0.6
          'cairo': 'cairo 1.14.*',  # 1.12.18
          'ffmpeg': 'ffmpeg >=3.2.3,<3.2.6',  # NA
          'fontconfig': 'fontconfig 2.12.*',  # 2.12.1
          'freetype': 'freetype 2.7',  # 2.5.5
          'geos': 'geos 3.5.1',  # 3.5.0
          'giflib': 'giflib 5.1.*',  # NA
          'glib': 'glib 2.51.*',  # 2.50.2
          'gmp': 'gmp 6.1.*', # 6.1.0
          'harfbuzz': 'harfbuzz 1.3.4',  # 0.9.39
          'hdf5': 'hdf5 1.8.18|1.8.18.*',  # 1.8.17
          'icu': 'icu 58.*',  # 54.1
          'jpeg': 'jpeg 9*',  # 9b (but defaults is probably using 8d)
          'krb5': 'krb5 1.14.*',  # 1.13.2
          'libblitz': 'libblitz 0.10|0.10.*',  # NA
          'libevent': 'libevent 2.0.22',  # 2.0.22
          'libgdal': 'libgdal 2.1.*',  # 2.1.0
          'libmatio': 'libmatio 1.5.*',  # NA
          'libnetcdf': 'libnetcdf 4.4.*',  # 4.4.1
          'libpng': 'libpng >=1.6.28,<1.7',  # 1.6.27
          'libssh2': 'libssh2 1.8.*',  # 1.8.0
          'libsvm': 'libsvm 3.21|3.21.*',  # NA
          'libtiff': 'libtiff >=4.0.3,<4.0.8',  # 4.0.6
          'libxml2': 'libxml2 2.9.*',  # 2.9.4
          'lzo': 'lzo 2.*',  # 2.06
          'metis': 'metis 5.1.*',  # NA
          'mpfr': 'mpfr 3.1.*',  # 3.1.5
          'ncurses': 'ncurses 5.9',  # 5.9
          'netcdf-cxx4': 'netcdf-cxx4 4.3.*',  # NA
          'netcdf-fortran': 'netcdf-fortran 4.4.*',  #
          'openblas': 'openblas 0.2.19|0.2.19.*',  # 0.2.19
          'openssl': 'openssl 1.0.*',  # 1.0.2k
          'pango': 'pango 1.40.*',  # 1.40.3
          'pixman': 'pixman 0.34.*',  # 0.34.0
          'proj4': 'proj4 4.9.3',  # 4.9.2
          'pyqt': 'pyqt 5.6.*',  # 5.6.0
          'qt': 'qt 5.6.*',  # 5.6.2
          'readline': 'readline 6.2*',  # 6.2
          'sox': 'sox 14.4.2',  # NA
          'sqlite': 'sqlite 3.13.*',  # 3.13.0
          'tk': 'tk 8.5.*',  # 8.5.18
          'vlfeat': 'vlfeat 0.9.20',  # NA
          'xz': 'xz 5.2.*',  # 5.2.2
          'zlib': 'zlib 1.2.8',  # 1.2.8
        }

parser = argparse.ArgumentParser(description='Propose a feedstock update.')
parser.add_argument('--feedstocks-dir', help="The location of the feedstocks.",
                    default="~/dev/conda-forge/feedstocks")
regexp_or_package = parser.add_mutually_exclusive_group()
regexp_or_package.add_argument('--regexp', help="Regexp of feedstocks to consider.")
regexp_or_package.add_argument('--package', help="A specific package to check.")
parser.add_argument('--local', help="Whether to use the existing local checkouts of the feedstocks, or to fetch from upstream", action="store_true", default=False)
parser.add_argument('--limit', help="Limit the number of packages to propose changes for (0 is unlimited).",
                    default=1, type=int)
args = parser.parse_args()

feedstocks_dir = os.path.expanduser(args.feedstocks_dir)
change_limit = args.limit

gh_token = conda_smithy.github.gh_token()
gh = github.Github(gh_token)

gh_me = gh.get_user()

if gh_me.login != 'conda-forge-admin':
    raise ValueError("The github token isn't that of conda-forge-admin (it's "
                     "for {}), I'm going to have to bail.".format(gh_me.login))

gh_forge = gh.get_organization('conda-forge')

if args.package:
    package_name = args.package
    package_feedstock = '{}-feedstock'.format(package_name)
    args.regexp = package_name

need_to_clone = args.local == False
feedstock_gen = feedstocks.feedstocks_yaml('conda-forge', feedstocks_dir, use_local=args.local,
                                           randomise=True, pull_up_to_date=need_to_clone, regexp=args.regexp)

def my_repos(gh_user):
    """
    List all of my repos.
    See https://github.com/PyGithub/PyGithub/issues/390 for rationale.

    """
    return github.PaginatedList.PaginatedList(
                github.Repository.Repository,
                gh_user._requester,
                gh_user.url + "/repos",
                dict(affiliation="owner"))

def list_pulls(repo, state='open', head=None):
    """
    List all of the pull requests that match the given criteria.

    At the time of writing, pygithub doesn't allow you to specify the head,
    so I had to create this function.

    """
    url_parameters = dict(state=state)
    if head:
        url_parameters['head'] = head
    return github.PaginatedList.PaginatedList(
        github.PullRequest.PullRequest,
        repo._requester,
        repo.url + "/pulls",
        url_parameters
    )

if args.package:
    try:
        my_repos = {package_feedstock: gh_me.get_repo(package_feedstock)}
    except github.UnknownObjectException:
        # We haven't forked it yet!
        my_repos = {}
    forge_repos = {package_feedstock: gh_forge.get_repo(package_feedstock)}
else:
    print("Collecting list of conda-forge-admin repos...")
    my_repos = {repo.name: repo for repo in my_repos(gh_me)}
    print("Collecting list of conda-forge repos...")
    forge_repos = {repo.name: repo for repo in gh_forge.get_repos()}

@contextmanager
def tmp_remote(repo, remote_name, url):
    if remote_name in [remote.name for remote in repo.remotes]:
        repo.delete_remote(remote_name)
    remote = repo.create_remote(remote_name, url)
    yield remote
    repo.delete_remote(remote_name)

@contextmanager
def create_update_pr(clone, remote_head, fork_remote, upstream_remote):
    target_branch = 'feedstock_version_pin_{}'.format(remote_head)
    if target_branch in clone.heads:
        # Detach the head
        clone.head.reference = clone.commit('upstream/master')
        clone.delete_head(target_branch, '-D')
    clone.create_head(target_branch, upstream_remote.refs[remote_head]).set_tracking_branch(upstream_remote.refs[remote_head])

    # Reset the working tree to a clean state.
    clone.head.reset(index=True, working_tree=True)
    clone.heads[target_branch].checkout()

    # It is at this point we pass context back to the caller so that they can
    # do whatever they like to the repo (like rerender the feedstock).
    context = []
    yield context

    # If nothing was done, don't need a PR!
    has_change = True
    if not clone.is_dirty():
        # We don't need this feedstock - it is slap-bang up to date. :)
        print("{} was checked, and is up-to-date".format(feedstock.name))
        has_change = False

    if has_change:
        clone.git.add('-A')
        author = git.Actor(gh_me.login, gh_me.email)
        commit = clone.index.commit("MNT: Updated some of the pinned versions",
                                    author=author)

        change_from_remote_branch = True
        full_ref = '{}/{}'.format(fork_remote.name, target_branch)

        if full_ref in [ref.name for ref in fork_remote.refs]:
            diff = commit.diff(fork_remote.refs[target_branch])
            if not diff:
                # There were no differences between this and the remote targt branch, so just continue.
                print("{} was checked, and whilst there are changes needed, the branch ({}) is up-to-date".format(feedstock.name, target_branch))
                change_from_remote_branch = False

        fork_remote.push('+{}'.format(target_branch))

        if change_from_remote_branch:
            fork_remote.push('+{}'.format(target_branch))

            pull_requests = list(list_pulls(forge_feedstock, state='open', head='{}:{}'.format(gh_me.login, target_branch)))

            if pull_requests:
                pull = pull_requests[0]
                msg = textwrap.dedent("""
    It's the friendly automated conda-forge-admin here again.

    Just to let you know, I've updated this PR so that it has the latest pinned versions.

    If there are no problems with it, please consider merging this PR.
    If there are concerns about it, please ping the 'conda-forge/core' team (using the @ notation in a comment).

    Thanks!
                       """)
                pull.create_issue_comment(msg)
                print('Updated PR on {}'.format(pull.html_url))
            else:
                msg = textwrap.dedent("""
    Hi! This is the friendly conda-forge-admin automated user.

    I've bumped some of the conda-forge pinned versions, and noticed that it impacts this feedstock.
    If the changes look good, then please go ahead and merge this PR.
    If you have any questions about the changes though, please feel free to ping the 'conda-forge/core' team (using the @ notation in a comment).

    Thanks!

                """)

                pull = forge_feedstock.create_pull(title='MNT: Update pinned versions.',
                                                   body=msg,
                                                   head="{}:{}".format(gh_me.login, target_branch), base=remote_head)
                print('Opened PR on {}'.format(pull.html_url))
            context.append(pull)

count = 0
for feedstock, git_ref, meta_content, recipe in feedstock_gen:
    if feedstock.name not in forge_repos:
        raise ValueError("There exists a feedstock ({}) which isn't in the "
                         "conda-forge org.".format(feedstock.name))

    if feedstock.name not in my_repos:
        forge_repo = gh_forge.get_repo(feedstock.name)
        print('Forking {}'.format(feedstock.name))
        my_repos[feedstock.name] = gh_me.create_fork(forge_repo)

    clone = git.Repo(feedstock.directory)
    admin_fork = my_repos[feedstock.name]
    forge_feedstock = forge_repos[feedstock.name]

    skip_after_package = False

    # Put an appropriate conda-forge-admin remote in place.
    with tmp_remote(clone, gh_me.login,
                    admin_fork.clone_url.replace('https://',
                                                 'https://{}@'.format(gh_token))) as remote:
        try:
            remote.fetch()
        except git.exc.GitCommandError as ex:
            # Periodically this fetch fails with a "git remote error: access denied
            # or repository not exported". Presumably this is because we have only just
            # forked it and GitHub is taking time to catch up, so we wait a few seconds
            # and try again.
            print("Sleeping for 10s before retrying due to the following error: \n", str(ex))
            time.sleep(10)
            remote.fetch()

        remote_branch = git_ref.remote_head.replace('{}/'.format(gh_me.login), '')
        with create_update_pr(clone, remote_branch, remote, clone.remotes['upstream']) as pr:
            replacements = {}
            for section_name in ['run', 'build']:
                requirements = recipe.get('requirements')
                if requirements is None:
                    break
                section = requirements.get(section_name)
                if not section:
                    continue

                for pos, dep in enumerate(section):
                    for name, pin in pinned.items():
                        if re.match(r"^\s*%s\s*$" % name, dep) and dep != pin:
                            replacements['- ' + str(dep)] = '- ' + pin
            if replacements:
                current_build_number = recipe['build']['number']
                replacements['number: {}'.format(current_build_number)] = 'number: {}'.format(current_build_number + 1)
            content = meta_content
            for orig, new in replacements.items():
                content = re.sub(
                    # Use capture groups to get the indentation correct.
                    r"(^\s*)%s(\s*)$" % orig,
                    r"\1%s\2" % new,
                    content,
                    flags=re.MULTILINE)
            forge_yaml = os.path.join(feedstock.directory, 'recipe', 'meta.yaml')
            with open(forge_yaml, 'w') as fh:
                fh.write(content)
        if pr:
            skip_after_package = True
    # Stop processing any more feedstocks until the next time the script is run.
    if skip_after_package:
        count += 1

    if change_limit > 0 and count >= change_limit:
        break
<commit_msg>udpdate openblas pin<commit_after>#!/usr/bin/env conda-execute

# conda execute
# env:
#  - python
#  - conda-smithy >=1.1
#  - gitpython
#  - pygithub
# channels:
#  - conda-forge
# run_with: python

import argparse
from contextlib import contextmanager
import os
import random
import re
import textwrap
import time

import git
import github

import conda_smithy.github
import conda_smithy.feedstocks as feedstocks

# The version commented right after the comma is the one available for **Linux** in **defaults**.
# That version is assumed to be the same in all platforms and the one used to build all packages.
# BE VERY CAREFUL when updating pinnings. Make sure to check https://abi-laboratory.pro/tracker/
# for any incompatibility between versions.
# Also be aware that currently the following pinnings are used for both build and run meta.yaml
# sections. Defining the same pinning for both sections means we need to be more restrictive.
#
# For example, we used to define gmp as 'gmp >=5.0.1,<7'. This is wrong as it allows a package
# to be built against gmp 6.1.0 but run with 5.0.1.
# As showed by https://abi-laboratory.pro/tracker/timeline/gmp/ binaries built with 6.1.0 are not
# compatible with 5.0.1 (the other way around would be true).
pinned = {
          'boost': 'boost 1.64.*',  # 1.61.0
          'boost-cpp': 'boost-cpp 1.64.*',  # NA
          'bzip2': 'bzip2 1.0.*',  # 1.0.6
          'cairo': 'cairo 1.14.*',  # 1.12.18
          'ffmpeg': 'ffmpeg >=3.2.3,<3.2.6',  # NA
          'fontconfig': 'fontconfig 2.12.*',  # 2.12.1
          'freetype': 'freetype 2.7',  # 2.5.5
          'geos': 'geos 3.5.1',  # 3.5.0
          'giflib': 'giflib 5.1.*',  # NA
          'glib': 'glib 2.51.*',  # 2.50.2
          'gmp': 'gmp 6.1.*', # 6.1.0
          'harfbuzz': 'harfbuzz 1.3.4',  # 0.9.39
          'hdf5': 'hdf5 1.8.18|1.8.18.*',  # 1.8.17
          'icu': 'icu 58.*',  # 54.1
          'jpeg': 'jpeg 9*',  # 9b (but defaults is probably using 8d)
          'krb5': 'krb5 1.14.*',  # 1.13.2
          'libblitz': 'libblitz 0.10|0.10.*',  # NA
          'libevent': 'libevent 2.0.22',  # 2.0.22
          'libgdal': 'libgdal 2.1.*',  # 2.1.0
          'libmatio': 'libmatio 1.5.*',  # NA
          'libnetcdf': 'libnetcdf 4.4.*',  # 4.4.1
          'libpng': 'libpng >=1.6.28,<1.7',  # 1.6.27
          'libssh2': 'libssh2 1.8.*',  # 1.8.0
          'libsvm': 'libsvm 3.21|3.21.*',  # NA
          'libtiff': 'libtiff >=4.0.3,<4.0.8',  # 4.0.6
          'libxml2': 'libxml2 2.9.*',  # 2.9.4
          'lzo': 'lzo 2.*',  # 2.06
          'metis': 'metis 5.1.*',  # NA
          'mpfr': 'mpfr 3.1.*',  # 3.1.5
          'ncurses': 'ncurses 5.9',  # 5.9
          'netcdf-cxx4': 'netcdf-cxx4 4.3.*',  # NA
          'netcdf-fortran': 'netcdf-fortran 4.4.*',  #
          'openblas': 'openblas 0.2.20|0.2.20.*',  # 0.2.19
          'openssl': 'openssl 1.0.*',  # 1.0.2k
          'pango': 'pango 1.40.*',  # 1.40.3
          'pixman': 'pixman 0.34.*',  # 0.34.0
          'proj4': 'proj4 4.9.3',  # 4.9.2
          'pyqt': 'pyqt 5.6.*',  # 5.6.0
          'qt': 'qt 5.6.*',  # 5.6.2
          'readline': 'readline 6.2*',  # 6.2
          'sox': 'sox 14.4.2',  # NA
          'sqlite': 'sqlite 3.13.*',  # 3.13.0
          'tk': 'tk 8.5.*',  # 8.5.18
          'vlfeat': 'vlfeat 0.9.20',  # NA
          'xz': 'xz 5.2.*',  # 5.2.2
          'zlib': 'zlib 1.2.8',  # 1.2.8
        }

parser = argparse.ArgumentParser(description='Propose a feedstock update.')
parser.add_argument('--feedstocks-dir', help="The location of the feedstocks.",
                    default="~/dev/conda-forge/feedstocks")
regexp_or_package = parser.add_mutually_exclusive_group()
regexp_or_package.add_argument('--regexp', help="Regexp of feedstocks to consider.")
regexp_or_package.add_argument('--package', help="A specific package to check.")
parser.add_argument('--local', help="Whether to use the existing local checkouts of the feedstocks, or to fetch from upstream", action="store_true", default=False)
parser.add_argument('--limit', help="Limit the number of packages to propose changes for (0 is unlimited).",
                    default=1, type=int)
args = parser.parse_args()

feedstocks_dir = os.path.expanduser(args.feedstocks_dir)
change_limit = args.limit

gh_token = conda_smithy.github.gh_token()
gh = github.Github(gh_token)

gh_me = gh.get_user()

if gh_me.login != 'conda-forge-admin':
    raise ValueError("The github token isn't that of conda-forge-admin (it's "
                     "for {}), I'm going to have to bail.".format(gh_me.login))

gh_forge = gh.get_organization('conda-forge')

if args.package:
    package_name = args.package
    package_feedstock = '{}-feedstock'.format(package_name)
    args.regexp = package_name

need_to_clone = args.local == False
feedstock_gen = feedstocks.feedstocks_yaml('conda-forge', feedstocks_dir, use_local=args.local,
                                           randomise=True, pull_up_to_date=need_to_clone, regexp=args.regexp)

def my_repos(gh_user):
    """
    List all of my repos.
    See https://github.com/PyGithub/PyGithub/issues/390 for rationale.

    """
    return github.PaginatedList.PaginatedList(
                github.Repository.Repository,
                gh_user._requester,
                gh_user.url + "/repos",
                dict(affiliation="owner"))

def list_pulls(repo, state='open', head=None):
    """
    List all of the pull requests that match the given criteria.

    At the time of writing, pygithub doesn't allow you to specify the head,
    so I had to create this function.

    """
    url_parameters = dict(state=state)
    if head:
        url_parameters['head'] = head
    return github.PaginatedList.PaginatedList(
        github.PullRequest.PullRequest,
        repo._requester,
        repo.url + "/pulls",
        url_parameters
    )

if args.package:
    try:
        my_repos = {package_feedstock: gh_me.get_repo(package_feedstock)}
    except github.UnknownObjectException:
        # We haven't forked it yet!
        my_repos = {}
    forge_repos = {package_feedstock: gh_forge.get_repo(package_feedstock)}
else:
    print("Collecting list of conda-forge-admin repos...")
    my_repos = {repo.name: repo for repo in my_repos(gh_me)}
    print("Collecting list of conda-forge repos...")
    forge_repos = {repo.name: repo for repo in gh_forge.get_repos()}

@contextmanager
def tmp_remote(repo, remote_name, url):
    if remote_name in [remote.name for remote in repo.remotes]:
        repo.delete_remote(remote_name)
    remote = repo.create_remote(remote_name, url)
    yield remote
    repo.delete_remote(remote_name)

@contextmanager
def create_update_pr(clone, remote_head, fork_remote, upstream_remote):
    target_branch = 'feedstock_version_pin_{}'.format(remote_head)
    if target_branch in clone.heads:
        # Detach the head
        clone.head.reference = clone.commit('upstream/master')
        clone.delete_head(target_branch, '-D')
    clone.create_head(target_branch, upstream_remote.refs[remote_head]).set_tracking_branch(upstream_remote.refs[remote_head])

    # Reset the working tree to a clean state.
    clone.head.reset(index=True, working_tree=True)
    clone.heads[target_branch].checkout()

    # It is at this point we pass context back to the caller so that they can
    # do whatever they like to the repo (like rerender the feedstock).
    context = []
    yield context

    # If nothing was done, don't need a PR!
    has_change = True
    if not clone.is_dirty():
        # We don't need this feedstock - it is slap-bang up to date. :)
        print("{} was checked, and is up-to-date".format(feedstock.name))
        has_change = False

    if has_change:
        clone.git.add('-A')
        author = git.Actor(gh_me.login, gh_me.email)
        commit = clone.index.commit("MNT: Updated some of the pinned versions",
                                    author=author)

        change_from_remote_branch = True
        full_ref = '{}/{}'.format(fork_remote.name, target_branch)

        if full_ref in [ref.name for ref in fork_remote.refs]:
            diff = commit.diff(fork_remote.refs[target_branch])
            if not diff:
                # There were no differences between this and the remote targt branch, so just continue.
                print("{} was checked, and whilst there are changes needed, the branch ({}) is up-to-date".format(feedstock.name, target_branch))
                change_from_remote_branch = False

        fork_remote.push('+{}'.format(target_branch))

        if change_from_remote_branch:
            fork_remote.push('+{}'.format(target_branch))

            pull_requests = list(list_pulls(forge_feedstock, state='open', head='{}:{}'.format(gh_me.login, target_branch)))

            if pull_requests:
                pull = pull_requests[0]
                msg = textwrap.dedent("""
    It's the friendly automated conda-forge-admin here again.

    Just to let you know, I've updated this PR so that it has the latest pinned versions.

    If there are no problems with it, please consider merging this PR.
    If there are concerns about it, please ping the 'conda-forge/core' team (using the @ notation in a comment).

    Thanks!
                       """)
                pull.create_issue_comment(msg)
                print('Updated PR on {}'.format(pull.html_url))
            else:
                msg = textwrap.dedent("""
    Hi! This is the friendly conda-forge-admin automated user.

    I've bumped some of the conda-forge pinned versions, and noticed that it impacts this feedstock.
    If the changes look good, then please go ahead and merge this PR.
    If you have any questions about the changes though, please feel free to ping the 'conda-forge/core' team (using the @ notation in a comment).

    Thanks!

                """)

                pull = forge_feedstock.create_pull(title='MNT: Update pinned versions.',
                                                   body=msg,
                                                   head="{}:{}".format(gh_me.login, target_branch), base=remote_head)
                print('Opened PR on {}'.format(pull.html_url))
            context.append(pull)

count = 0
for feedstock, git_ref, meta_content, recipe in feedstock_gen:
    if feedstock.name not in forge_repos:
        raise ValueError("There exists a feedstock ({}) which isn't in the "
                         "conda-forge org.".format(feedstock.name))

    if feedstock.name not in my_repos:
        forge_repo = gh_forge.get_repo(feedstock.name)
        print('Forking {}'.format(feedstock.name))
        my_repos[feedstock.name] = gh_me.create_fork(forge_repo)

    clone = git.Repo(feedstock.directory)
    admin_fork = my_repos[feedstock.name]
    forge_feedstock = forge_repos[feedstock.name]

    skip_after_package = False

    # Put an appropriate conda-forge-admin remote in place.
    with tmp_remote(clone, gh_me.login,
                    admin_fork.clone_url.replace('https://',
                                                 'https://{}@'.format(gh_token))) as remote:
        try:
            remote.fetch()
        except git.exc.GitCommandError as ex:
            # Periodically this fetch fails with a "git remote error: access denied
            # or repository not exported". Presumably this is because we have only just
            # forked it and GitHub is taking time to catch up, so we wait a few seconds
            # and try again.
            print("Sleeping for 10s before retrying due to the following error: \n", str(ex))
            time.sleep(10)
            remote.fetch()

        remote_branch = git_ref.remote_head.replace('{}/'.format(gh_me.login), '')
        with create_update_pr(clone, remote_branch, remote, clone.remotes['upstream']) as pr:
            replacements = {}
            for section_name in ['run', 'build']:
                requirements = recipe.get('requirements')
                if requirements is None:
                    break
                section = requirements.get(section_name)
                if not section:
                    continue

                for pos, dep in enumerate(section):
                    for name, pin in pinned.items():
                        if re.match(r"^\s*%s\s*$" % name, dep) and dep != pin:
                            replacements['- ' + str(dep)] = '- ' + pin
            if replacements:
                current_build_number = recipe['build']['number']
                replacements['number: {}'.format(current_build_number)] = 'number: {}'.format(current_build_number + 1)
            content = meta_content
            for orig, new in replacements.items():
                content = re.sub(
                    # Use capture groups to get the indentation correct.
                    r"(^\s*)%s(\s*)$" % orig,
                    r"\1%s\2" % new,
                    content,
                    flags=re.MULTILINE)
            forge_yaml = os.path.join(feedstock.directory, 'recipe', 'meta.yaml')
            with open(forge_yaml, 'w') as fh:
                fh.write(content)
        if pr:
            skip_after_package = True
    # Stop processing any more feedstocks until the next time the script is run.
    if skip_after_package:
        count += 1

    if change_limit > 0 and count >= change_limit:
        break
<|endoftext|>


<commit_before>#!/usr/bin/python
# -*- coding: utf-8 -*-

# Copyright (c) 2015, Ricardo Sánchez-Sáez. All rights reserved.
#
# Redistribution and use in source and binary forms, with or without modification,
# are permitted provided that the following conditions are met:
#
# 1.  Redistributions of source code must retain the above copyright notice, this
# list of conditions and the following disclaimer.
#
# 2.  Redistributions in binary form must reproduce the above copyright notice,
# this list of conditions and the following disclaimer in the documentation and/or
# other materials provided with the distribution.
#
# 3.  Neither the name of the copyright holder(s) nor the names of any contributors
# may be used to endorse or promote products derived from this software without
# specific prior written permission. No license is granted to the trademarks of
# the copyright holders even if such marks are included in this software.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE
# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

# This script reads and converts a binary .strings file (binary .plist) into a plaintext .string file.
#
# Since binary .strings files are generally unsorted, it takes a master plaintext .strings file
# (typically en.lproj/Localizable.strings) and uses it for sorting they keys in the target converted file.
#
# The target file can have all or some of the string keys in the master file. If the master file has
# additional string keys not found in the target file, they are ignored.
# String keys present in the target file but not present in the master file are added at the end of the
# converted file in an undefined order.
#
# This script has been tested with Python 2.7.6
#
# Single file usage example:
#   ./plist-to-strings.py -m en.lproj/Localizable.strings -t es.lproj/Localizable.strings
#
# Whole project usage example (using 'en.lproj' as the master plaintext localization):
#   localizedFolder="../ProjectPath/Localized"; for languageFolder in $(ls $localizedFolder); do if [[ "$languageFolder" != "en.lproj" ]]; then ./plist-to-strings.py -m ${localizedFolder}/en.lproj/Localizable.strings -t ${localizedFolder}/${languageFolder}/Localizable.strings; fi; done

import argparse
import os
import plistlib
import shutil
import subprocess
import sys
import uuid

def warning(string):
    sys.stdout.write("WARNING: " + string + '\n\n')

def error(string):
    sys.stderr.write("ERROR: " + string + '\n\n')

parser = argparse.ArgumentParser()
parser.add_argument('-m', '--master', help='Master plain text .strings file', required=True)
parser.add_argument('-t', '--target', help='Target binary plist .strings file to convert to plain text', required=True)
args = parser.parse_args()
masterFileName = args.master
targetFileName = args.target

def convertPlistToStrings(masterFileName, targetFileName):
    def getStringsLine(stringKey, localizedString):
        # escape \, ", \n and \r characters for the target plain text string
        escapedString = localizedString.replace('\\', '\\\\').replace('"','\\"').replace('\n','\\n').replace('\r','\\r')
        return '"' + stringKey + '" = "' + escapedString + '";\n'

    if not os.path.isfile(masterFileName):
        print('Master .strings file not found: ' + masterFileName + '\n')
        parser.print_usage()
        sys.exit(1)

    if not os.path.isfile(targetFileName):
        print('Target .strings file not found: ' + targetFileName + '\n')
        parser.print_usage()
        sys.exit(1)

    print('Converting: ' + targetFileName)

    outputEncoding = 'UTF-8'

    targetFileNameWithoutExtension = os.path.splitext(targetFileName)[0]
    UUID = uuid.uuid1()
    temporaryTargetFileName = targetFileNameWithoutExtension + '.' + UUID.hex + '.strings.plist'
    shutil.copyfile(targetFileName, temporaryTargetFileName)

    # convert binary .strings.plist to hex .strings.plist
    subprocess.call(['plutil', '-convert', 'xml1', temporaryTargetFileName])
    targetPlist = plistlib.readPlist(temporaryTargetFileName)

    masterFile = open(masterFileName)

    temporaryOutputFileName = targetFileNameWithoutExtension + '.' + UUID.hex + '.strings'
    temporaryOutputFile = open(temporaryOutputFileName, 'w')

    for line in masterFile:
        # preserve comments and blank lines
        # warning: multiline comments cannot have a line starting with the double quote character "
        if len(line.strip())==0 or line.strip()[0] != '"':
            temporaryOutputFile.write(line.encode(outputEncoding))

        else:
            # get string key
            # line example:             "CONSENT_NAME_TITLE" = "Consent";
            # tokenizedLine example:    ['', 'CONSENT_NAME_TITLE', ' = ', 'Consent', ';']

            # remove escaped \\ and \" so they don't interfere with the tokenizer (we don't need the correct master string)
            intermediateLine = line.strip().replace('\\\\', '').replace('\\"','')
            tokenizedLine = intermediateLine.split('"')

            # ignore abnormal lines (these should not appear)
            if len(tokenizedLine) != 5:
                error("Ignoring malformed line:" + line.strip())
                continue

            stringKey = tokenizedLine[1]
            if stringKey not in targetPlist:
                warning("String key not found in target file: " + stringKey)
                continue

            targetLine = getStringsLine(stringKey, targetPlist[stringKey])
            temporaryOutputFile.write(targetLine.encode(outputEncoding))

            del targetPlist[stringKey]

    # Add remaining target keys
    if len(targetPlist) > 0:
        temporaryOutputFile.write("\n\n/* Unsorted */\n")
        for stringKey in targetPlist:
            targetLine = getStringsLine(stringKey, targetPlist[stringKey]).encode(outputEncoding)
            temporaryOutputFile.write(targetLine)

    temporaryOutputFile.close()
    masterFile.close()

    os.remove(temporaryTargetFileName)
    os.rename(temporaryOutputFileName, targetFileName)

    print(' done')

convertPlistToStrings(masterFileName, targetFileName)
<commit_msg>plist-to-strings.py: move argument checks and logging outside function<commit_after>#!/usr/bin/python
# -*- coding: utf-8 -*-

# Copyright (c) 2015, Ricardo Sánchez-Sáez. All rights reserved.
#
# Redistribution and use in source and binary forms, with or without modification,
# are permitted provided that the following conditions are met:
#
# 1.  Redistributions of source code must retain the above copyright notice, this
# list of conditions and the following disclaimer.
#
# 2.  Redistributions in binary form must reproduce the above copyright notice,
# this list of conditions and the following disclaimer in the documentation and/or
# other materials provided with the distribution.
#
# 3.  Neither the name of the copyright holder(s) nor the names of any contributors
# may be used to endorse or promote products derived from this software without
# specific prior written permission. No license is granted to the trademarks of
# the copyright holders even if such marks are included in this software.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE
# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

# This script reads and converts a binary .strings file (binary .plist) into a plaintext .string file.
#
# Since binary .strings files are generally unsorted, it takes a master plaintext .strings file
# (typically en.lproj/Localizable.strings) and uses it for sorting they keys in the target converted file.
#
# The target file can have all or some of the string keys in the master file. If the master file has
# additional string keys not found in the target file, they are ignored.
# String keys present in the target file but not present in the master file are added at the end of the
# converted file in an undefined order.
#
# This script has been tested with Python 2.7.6
#
# Single file usage example:
#   ./plist-to-strings.py -m en.lproj/Localizable.strings -t es.lproj/Localizable.strings
#
# Whole project usage example (using 'en.lproj' as the master plaintext localization):
#   localizedFolder="../ProjectPath/Localized"; for languageFolder in $(ls $localizedFolder); do if [[ "$languageFolder" != "en.lproj" ]]; then ./plist-to-strings.py -m ${localizedFolder}/en.lproj/Localizable.strings -t ${localizedFolder}/${languageFolder}/Localizable.strings; fi; done

import argparse
import os
import plistlib
import shutil
import subprocess
import sys
import uuid

def warning(string):
    sys.stdout.write("WARNING: " + string + '\n\n')

def error(string):
    sys.stderr.write("ERROR: " + string + '\n\n')

parser = argparse.ArgumentParser()
parser.add_argument('-m', '--master', help='Master plain text .strings file', required=True)
parser.add_argument('-t', '--target', help='Target binary plist .strings file to convert to plain text', required=True)
args = parser.parse_args()
masterFileName = args.master
targetFileName = args.target

if not os.path.isfile(masterFileName):
    print('Master .strings file not found: ' + masterFileName + '\n')
    parser.print_usage()
    sys.exit(1)

if not os.path.isfile(targetFileName):
    print('Target .strings file not found: ' + targetFileName + '\n')
    parser.print_usage()
    sys.exit(1)

def convertBinaryPlistToStrings(masterFileName, targetFileName):
    def getStringsLine(stringKey, localizedString):
        # escape '\', '"', '\n' and '\r' characters on the target plain text string
        escapedString = localizedString.replace('\\', '\\\\').replace('"','\\"').replace('\n','\\n').replace('\r','\\r')
        return '"' + stringKey + '" = "' + escapedString + '";\n'

    outputEncoding = 'UTF-8'

    targetFileNameWithoutExtension = os.path.splitext(targetFileName)[0]
    UUID = uuid.uuid1()
    temporaryTargetFileName = targetFileNameWithoutExtension + '.' + UUID.hex + '.strings.plist'
    shutil.copyfile(targetFileName, temporaryTargetFileName)

    # convert binary .strings.plist to hex .strings.plist
    subprocess.call(['plutil', '-convert', 'xml1', temporaryTargetFileName])
    targetPlist = plistlib.readPlist(temporaryTargetFileName)

    masterFile = open(masterFileName)

    temporaryOutputFileName = targetFileNameWithoutExtension + '.' + UUID.hex + '.strings'
    temporaryOutputFile = open(temporaryOutputFileName, 'w')

    for line in masterFile:
        # preserve comments and blank lines
        # warning: multiline comments cannot have a line starting with the double quote character "
        if len(line.strip())==0 or line.strip()[0] != '"':
            temporaryOutputFile.write(line.encode(outputEncoding))

        else:
            # get string key
            # line example:             "CONSENT_NAME_TITLE" = "Consent";
            # tokenizedLine example:    ['', 'CONSENT_NAME_TITLE', ' = ', 'Consent', ';']

            # remove escaped \\ and \" so they don't interfere with the tokenizer (we don't need the correct master string)
            intermediateLine = line.strip().replace('\\\\', '').replace('\\"','')
            tokenizedLine = intermediateLine.split('"')

            # ignore abnormal lines (these should not appear)
            if len(tokenizedLine) != 5:
                error("Ignoring malformed line:" + line.strip())
                continue

            stringKey = tokenizedLine[1]
            if stringKey not in targetPlist:
                warning("String key not found in target file: " + stringKey)
                continue

            targetLine = getStringsLine(stringKey, targetPlist[stringKey])
            temporaryOutputFile.write(targetLine.encode(outputEncoding))

            del targetPlist[stringKey]

    # Add remaining target keys
    if len(targetPlist) > 0:
        temporaryOutputFile.write("\n\n/* Unsorted */\n")
        for stringKey in targetPlist:
            targetLine = getStringsLine(stringKey, targetPlist[stringKey]).encode(outputEncoding)
            temporaryOutputFile.write(targetLine)

    temporaryOutputFile.close()
    masterFile.close()

    os.remove(temporaryTargetFileName)
    os.rename(temporaryOutputFileName, targetFileName)

print('Converting: ' + targetFileName)
convertBinaryPlistToStrings(masterFileName, targetFileName)
print('...done')
<|endoftext|>


<commit_before>#!/usr/bin/env python
"""
                     The LLVM Compiler Infrastructure

This file is distributed under the University of Illinois Open Source
License. See LICENSE.TXT for details.

Prepares language bindings for LLDB build process.  Run with --help
to see a description of the supported command line arguments.
"""

# Python modules:
import argparse
import logging
import os
import sys

def prepare_binding_for_language(scripts_dir, script_lang, options):
    """Prepares the binding for a specific language.

    @param scripts_dir the full path to the scripts source directory.
    @param script_lang the name of the script language.  Should be a child
    directory within the scripts dir, and should contain a
    prepare_scripts_{script_lang}.py script file in it.
    @param options the dictionary of parsed command line options.

    There is no return value.  If it returns, the process succeeded; otherwise,
    the process will exit where it fails.
    """
    # Ensure the language-specific prepare module exists.
    script_name = "prepare_binding_{}.py".format(script_lang)
    lang_path = os.path.join(scripts_dir, script_lang)
    script_path = os.path.join(lang_path, script_name)
    if not os.path.exists(script_path):
        logging.error(
            "failed to find prepare script for language '%s' at '%s'",
            script_lang,
            script_path)
        sys.exit(-9)

    # Include this language-specific directory in the Python search
    # path.
    sys.path.append(os.path.normcase(lang_path))

    # Execute the specific language script
    module_name = os.path.splitext(script_name)[0]
    module = __import__(module_name)
    module.main(options)

    # Remove the language-specific directory from the Python search path.
    sys.path.remove(os.path.normcase(lang_path))

def prepare_all_bindings(options):
    """Prepares bindings for each of the languages supported.

    @param options the parsed arguments from the command line

    @return the exit value for the program. 0 is success, all othes
    indicate some kind of failure.
    """
    # Check for the existence of the SWIG scripts folder
    scripts_dir = os.path.join(options.src_root, "scripts")
    if not os.path.exists(scripts_dir):
        logging.error("failed to find scripts dir: '%s'", scripts_dir)
        sys.exit(-8)

    # Collect list of child directories.  We expect there to be one
    # for each supported script language.
    child_dirs = [f for f in os.listdir(scripts_dir)
                  if os.path.isdir(os.path.join(scripts_dir, f))]

    # Remove directories that do not represent script languages.
    for removal_dir in [".svn", "interface", "__pycache__", "sphinx"]:
        if removal_dir in child_dirs:
            child_dirs.remove(removal_dir)

    logging.info("found script directories: %s", child_dirs)

    # Iterate script directory find any script language directories
    for script_lang in child_dirs:
        logging.info("executing language script for: '%s'", script_lang)
        prepare_binding_for_language(scripts_dir, script_lang, options)

def process_args(args):
    """Returns options processed from the provided command line.

    @param args the command line to process.
    """

    # Setup the parser arguments that are accepted.
    parser = argparse.ArgumentParser(
        description="Prepare language bindings for LLDB build.")

    # Arguments to control logging verbosity.
    parser.add_argument(
        "--debug", "-d",
        action="store_true",
        help="Set program logging level to DEBUG.")
    parser.add_argument(
        "--verbose", "-v",
        action="count",
        default=0,
        help=(
            "Increase logging verbosity level.  Default: only error and "
            "higher are displayed.  Each -v increases level of verbosity."))

    # Arguments to control whether we're building an OS X-style
    # framework.  This is the opposite of the older "-m" (makefile)
    # option.
    parser.add_argument(
        "--config-build-dir",
        "--cfgBldDir",
        help=(
            "Configuration build dir, will use python module path "
            "if unspecified."))
    parser.add_argument(
        "--framework",
        action="store_true",
        help="Prepare as OS X-style framework.")
    parser.add_argument(
        "--generate-dependency-file",
        "-M",
        action="store_true",
        help="Make the dependency (.d) file for the wrappers.")
    parser.add_argument(
        "--prefix",
        help="Override path where the LLDB module is placed.")
    parser.add_argument(
        "--src-root",
        "--srcRoot",
        "-s",
        # Default to the parent directory of this script's directory.
        default=os.path.abspath(
            os.path.join(
                os.path.dirname(os.path.realpath(__file__)),
                os.path.pardir)),
        help="Specifies the LLDB source root directory.")
    parser.add_argument(
        "--swig-executable",
        "--swigExecutable",
        help="Path to the swig executable.")
    parser.add_argument(
        "--target-dir",
        "--targetDir",
        required=True,
        help=(
            "Specifies the build dir where the language binding "
            "should be placed"))

    # Process args.
    options = parser.parse_args(args)

    # Set logging level based on verbosity count.
    if options.debug:
        log_level = logging.DEBUG
    else:
        # See logging documentation for error levels.  We'll default
        # to showing ERROR or higher error messages.  For each -v
        # specified, we'll shift to the next lower-priority log level.
        log_level = logging.ERROR - 10 * options.verbose
        if log_level < logging.NOTSET:
            # Displays all logged messages.
            log_level = logging.NOTSET
    logging.basicConfig(level=log_level)
    logging.info("logging is using level: %d", log_level)

    return options

def main(args):
    """Drives the main script preparation steps.

    @param args list of command line arguments.
    """
    # Process command line arguments.
    options = process_args(args)
    logging.debug("Processed args: options=%s", options)

    # Check if the swig file exists.
    swig_path = os.path.normcase(
        os.path.join(options.src_root, "scripts", "lldb.swig"))
    if not os.path.isfile(swig_path):
        logging.error("swig file not found at '%s'", swig_path)
        sys.exit(-3)

    # Prepare bindings for each supported language binding.
    # This will error out if it doesn't succeed.
    prepare_all_bindings(options)
    sys.exit(0)

if __name__ == "__main__":
    # Run the main driver loop.
    main(sys.argv[1:])
<commit_msg>Added --find-swig option to allow searching for swig in path or other likely locations.<commit_after>#!/usr/bin/env python
"""
                     The LLVM Compiler Infrastructure

This file is distributed under the University of Illinois Open Source
License. See LICENSE.TXT for details.

Prepares language bindings for LLDB build process.  Run with --help
to see a description of the supported command line arguments.
"""

# Python modules:
import argparse
import logging
import os
import platform
import sys

def prepare_binding_for_language(scripts_dir, script_lang, options):
    """Prepares the binding for a specific language.

    @param scripts_dir the full path to the scripts source directory.
    @param script_lang the name of the script language.  Should be a child
    directory within the scripts dir, and should contain a
    prepare_scripts_{script_lang}.py script file in it.
    @param options the dictionary of parsed command line options.

    There is no return value.  If it returns, the process succeeded; otherwise,
    the process will exit where it fails.
    """
    # Ensure the language-specific prepare module exists.
    script_name = "prepare_binding_{}.py".format(script_lang)
    lang_path = os.path.join(scripts_dir, script_lang)
    script_path = os.path.join(lang_path, script_name)
    if not os.path.exists(script_path):
        logging.error(
            "failed to find prepare script for language '%s' at '%s'",
            script_lang,
            script_path)
        sys.exit(-9)

    # Include this language-specific directory in the Python search
    # path.
    sys.path.append(os.path.normcase(lang_path))

    # Execute the specific language script
    module_name = os.path.splitext(script_name)[0]
    module = __import__(module_name)
    module.main(options)

    # Remove the language-specific directory from the Python search path.
    sys.path.remove(os.path.normcase(lang_path))

def prepare_all_bindings(options):
    """Prepares bindings for each of the languages supported.

    @param options the parsed arguments from the command line

    @return the exit value for the program. 0 is success, all othes
    indicate some kind of failure.
    """
    # Check for the existence of the SWIG scripts folder
    scripts_dir = os.path.join(options.src_root, "scripts")
    if not os.path.exists(scripts_dir):
        logging.error("failed to find scripts dir: '%s'", scripts_dir)
        sys.exit(-8)

    # Collect list of child directories.  We expect there to be one
    # for each supported script language.
    child_dirs = [f for f in os.listdir(scripts_dir)
                  if os.path.isdir(os.path.join(scripts_dir, f))]

    # Remove directories that do not represent script languages.
    for removal_dir in [".svn", "interface", "__pycache__", "sphinx"]:
        if removal_dir in child_dirs:
            child_dirs.remove(removal_dir)

    logging.info("found script directories: %s", child_dirs)

    # Iterate script directory find any script language directories
    for script_lang in child_dirs:
        logging.info("executing language script for: '%s'", script_lang)
        prepare_binding_for_language(scripts_dir, script_lang, options)

def process_args(args):
    """Returns options processed from the provided command line.

    @param args the command line to process.
    """

    # Setup the parser arguments that are accepted.
    parser = argparse.ArgumentParser(
        description="Prepare language bindings for LLDB build.")

    # Arguments to control logging verbosity.
    parser.add_argument(
        "--debug", "-d",
        action="store_true",
        help="Set program logging level to DEBUG.")
    parser.add_argument(
        "--verbose", "-v",
        action="count",
        default=0,
        help=(
            "Increase logging verbosity level.  Default: only error and "
            "higher are displayed.  Each -v increases level of verbosity."))

    # Arguments to control whether we're building an OS X-style
    # framework.  This is the opposite of the older "-m" (makefile)
    # option.
    parser.add_argument(
        "--config-build-dir",
        "--cfgBldDir",
        help=(
            "Configuration build dir, will use python module path "
            "if unspecified."))
    parser.add_argument(
        "--find-swig",
        action="store_true",
        help=(
            "Indicates the swig executable should be searched for "
            "if not eplicitly provided.  Either this or the explicit "
            "swig executable option must be provided."))
    parser.add_argument(
        "--framework",
        action="store_true",
        help="Prepare as OS X-style framework.")
    parser.add_argument(
        "--generate-dependency-file",
        "-M",
        action="store_true",
        help="Make the dependency (.d) file for the wrappers.")
    parser.add_argument(
        "--prefix",
        help="Override path where the LLDB module is placed.")
    parser.add_argument(
        "--src-root",
        "--srcRoot",
        "-s",
        # Default to the parent directory of this script's directory.
        default=os.path.abspath(
            os.path.join(
                os.path.dirname(os.path.realpath(__file__)),
                os.path.pardir)),
        help="Specifies the LLDB source root directory.")
    parser.add_argument(
        "--swig-executable",
        "--swigExecutable",
        help="Path to the swig executable.")
    parser.add_argument(
        "--target-dir",
        "--targetDir",
        required=True,
        help=(
            "Specifies the build dir where the language binding "
            "should be placed"))

    # Process args.
    options = parser.parse_args(args)

    # Set logging level based on verbosity count.
    if options.debug:
        log_level = logging.DEBUG
    else:
        # See logging documentation for error levels.  We'll default
        # to showing ERROR or higher error messages.  For each -v
        # specified, we'll shift to the next lower-priority log level.
        log_level = logging.ERROR - 10 * options.verbose
        if log_level < logging.NOTSET:
            # Displays all logged messages.
            log_level = logging.NOTSET
    logging.basicConfig(level=log_level)
    logging.info("logging is using level: %d", log_level)

    return options

def find_file_in_paths(paths, exe_basename):
    """Returns the full exe path for the first path match.

    @params paths the list of directories to search for the exe_basename
    executable
    @params exe_basename the name of the file for which to search.
    e.g. "swig" or "swig.exe".

    @return the full path to the executable if found in one of the
    given paths; otherwise, returns None.
    """
    for path in paths:
        trial_exe_path = os.path.join(path, exe_basename)
        if os.path.exists(trial_exe_path):
            return os.path.normcase(trial_exe_path)
    return None

def find_swig_executable(options):
    """Finds the swig executable in the PATH or known good locations.

    Replaces options.swig_executable with the full swig executable path.
    """
    # Figure out what we're looking for.
    if platform.system() == 'Windows':
        exe_basename = "swig.exe"
        extra_dirs = []
    else:
        exe_basename = "swig"
        extra_dirs = ["/usr/local/bin"]

    # Figure out what paths to check.
    path_env = os.environ.get("PATH", None)
    if path_env is not None:
        paths_to_check = path_env.split(os.path.pathsep)
    else:
        paths_to_check = []

    # Add in the extra dirs
    paths_to_check.extend(extra_dirs)
    if len(paths_to_check) < 1:
        logging.error(
            "swig executable was not specified, PATH has no "
            "contents, and there are no extra directories to search")
        sys.exit(-6)

    # Find the swig executable
    options.swig_executable = find_file_in_paths(paths_to_check, exe_basename)
    if not options.swig_executable or len(options.swig_executable) < 1:
        logging.error(
            "failed to find exe='%s' in paths='%s'",
            exe_basename,
            paths_to_check)
        sys.exit(-6)
    logging.info("found swig executable: %s", options.swig_executable)

def main(args):
    """Drives the main script preparation steps.

    @param args list of command line arguments.
    """
    # Process command line arguments.
    options = process_args(args)
    logging.debug("Processed args: options=%s", options)

    # Ensure we have a swig executable.
    if not options.swig_executable or len(options.swig_executable) == 0:
        if options.find_swig:
            find_swig_executable(options)
        else:
            logging.error(
                "The --find-swig option must be specified "
                "when the swig executable location is not "
                "explicitly provided.")
            sys.exit(-12)

    # Check if the swig file exists.
    swig_path = os.path.normcase(
        os.path.join(options.src_root, "scripts", "lldb.swig"))
    if not os.path.isfile(swig_path):
        logging.error("swig file not found at '%s'", swig_path)
        sys.exit(-3)

    # Prepare bindings for each supported language binding.
    # This will error out if it doesn't succeed.
    prepare_all_bindings(options)
    sys.exit(0)

if __name__ == "__main__":
    # Run the main driver loop.
    main(sys.argv[1:])
<|endoftext|>


<commit_before>import argparse
import os
import pathlib
import sys
from subprocess import STDOUT, check_call

if sys.platform != "darwin":
    raise NotImplementedError

parser = argparse.ArgumentParser()
parser.add_argument(
    "--application-id",
    required=True,
    help="Certificate ID (should be added to the keychain).",
)
args = parser.parse_args()

path = pathlib.Path(__file__).parent.absolute()
dvc = path / "dist" / "dvc"
for root, _, fnames in os.walk(dvc):
    for fname in fnames:
        fpath = os.path.join(root, fname)
        print(f"signing {fpath}")
        check_call(
            [
                "codesign",
                "--force",
                "--verbose",
                "-s",
                args.application_id,
                "-o",
                "runtime",
                "--entitlements",
                "entitlements.plist",
                fpath,
            ],
            stderr=STDOUT,
            timeout=10,
        )
<commit_msg>sign: increase timeout<commit_after>import argparse
import os
import pathlib
import sys
from subprocess import STDOUT, check_call

if sys.platform != "darwin":
    raise NotImplementedError

parser = argparse.ArgumentParser()
parser.add_argument(
    "--application-id",
    required=True,
    help="Certificate ID (should be added to the keychain).",
)
args = parser.parse_args()

path = pathlib.Path(__file__).parent.absolute()
dvc = path / "dist" / "dvc"
for root, _, fnames in os.walk(dvc):
    for fname in fnames:
        fpath = os.path.join(root, fname)
        print(f"signing {fpath}")
        check_call(
            [
                "codesign",
                "--force",
                "--verbose",
                "-s",
                args.application_id,
                "-o",
                "runtime",
                "--entitlements",
                "entitlements.plist",
                fpath,
            ],
            stderr=STDOUT,
            timeout=30,
        )
<|endoftext|>


<commit_before>"""
		@file		qlibutils.py
		@author		xy
		@since		2012-07-23

		@brief		qLib-related utility functions.

		Location: $HIH/scripts/python/

"""

import hou

import glob
import os
import platform
import re
import subprocess
import traceback

def is_platform(name='none'):
	return name.lower() in platform.system().lower()

def is_linux(): return is_platform('linux')
def is_windows(): return is_platform('win')
def is_mac(): return is_platform('mac')

def statmsg(msg, warn=False):
	'''.'''
	s = hou.severityType.Warning if warn else hou.severityType.Message
	if hou.isUIAvailable():
		hou.ui.setStatusMessage(msg, severity=s)

def set_namespace_aliases(prefix="qLib::", alias=True, verbose=False):
	"""
	Defines (non-)namespaced aliases for operators with a particular namespace prefix.

	This is used for always creating the namespaced versions of assets, even if an
	older .hip file contains non-namespaced asset names.

	Mapping looks like:  <opname>  -->  <prefix>::<opname>::<version>

	@note
		IMPORTANT: Although the manual says it's fine to omit the version of a
		namespaced asset (and that would refer to the latest version),
		omitting it results in files getting messed up when loaded,
		so version numbers _are_ included in the opaliases.

	@note
		This function should be called (preferably) on Houdini startup, e.g.

		import qlibutils
		qlibutils.set_namespace_aliases( ["qLib::", "myStuff::"] )

	@todo
		For each asset, the highest version number should be found and used.
		Right now it uses the first version it founds (which is fine for now).

	"""

	if type(prefix) is list:
		for p in prefix: set_namespace_aliases(p)
		return

	assert "::" in prefix, "Include trailing '::' characters in prefix"

	cmds = []
	for file in hou.hda.loadedFiles():

		names = [ (d.nodeType().name(), d.nodeTypeCategory().name()) \
			for d in list(hou.hda.definitionsInFile(file)) \
			if prefix in d.nodeType().name() ]

		for n in names:

			try:
				# strip namespace prefix and version suffix
				old = re.sub("^[^:]+::", "", n[0])
				old = re.search("^[^:]+", old).group(0)

				# opalias <network> <namespaced-op.> <plain-old-op.>
				cmd = "opalias %s %s %s" % (n[1], n[0], old)

				if cmd not in cmds:
					if verbose: print cmd
					if alias: hou.hscript(cmd)
					cmds.append(cmd)
				else:
					print "# ALREADY ALIASED: %s (%s)" % (cmd, file)

			except:
				print "ERROR: %s" % traceback.format_exc()

def do_crash_recovery(calledFromUI=False):
	tmpdir = str(hou.getenv("TEMP"))
	files = glob.glob( os.path.join(tmpdir, '*.hip') )

	uicall = calledFromUI

	if hou.isUIAvailable() and len(files)>0:

		td = os.path.join(tmpdir, '') # dir with '/'
		files = [ (f, os.path.getmtime(f), ) for f in files ]
		files = sorted(files, key=lambda f: f[1], reverse=True)
		files = [ str(re.sub('^%s' % td, '', f[0])) for f in files ]

		sel = hou.ui.selectFromList(files, exclusive=True,
			title="Crash Recovery",
			message="Select .hip File to Recover")

		recovered = False
		if len(sel)>0:
			f = files[sel[0]]
			fn = os.path.join(tmpdir, f)

			# extract HIPNAME
			f = re.sub('^crash.', '', f)
			f = re.sub('\..+_[0-9]+\.hip', '.hip', f)

			# do recovery
			try:
				hou.hipFile.clear(True)
				hou.hipFile.load(fn, True)
				hou.setUpdateMode(hou.updateMode.Manual)
				recovered = True
			except:
				hou.ui.setStatusMessage("error while recovering file %s" % fn, hou.severityType.Error)
				print "ERROR: %s" % traceback.format_exc()

			hou.hipFile.setName(f)

		# delete crash file(s)

		msg = 'Cleanup: Delete all crash recovery hip files?'
		if recovered:
			msg = \
				'File recovered. Make sure to save it to a safe location.\n' \
				'NOTE: Update mode is set to "Manual" to avoid potential re-crashes.\n' \
				'\n%s' % msg

		d = hou.ui.displayMessage(msg, buttons=("DELETE", "Skip", ))
		if d==0:
			files = \
				glob.glob( os.path.join(tmpdir, 'crash.*') ) + \
				glob.glob( os.path.join(tmpdir, '*.hip') )
			for f in files:
			    try:
				os.remove(f)
			    except:
				pass

			hou.ui.setStatusMessage("crash recovery cleanup: deleted %d files" % len(files))
		else:
			pass # user cancelled

	else:
		# no crash files found
		#
		if uicall:
			hou.ui.setStatusMessage("  Crash Recovery:  No emergency-saved .hip file(s) found -- nothing to recover.", hou.severityType.ImportantMessage)
			pass

def open_dir(dir="", env=None):
	'''.'''
	dir=str(dir)

	if env:
		dir = str(hou.getenv(env))

	if not os.path.exists(dir):
		statmsg("Directory doesn't exist (%s)" % dir, warn=True)
		return

	if is_linux():
		statmsg("(linux) xdg-open %s" % dir)
		subprocess.call(["xdg-open", dir])

	if is_windows():
		dir = dir.replace('/', '\\')
		statmsg("(windows) start %s" % dir)
		subprocess.call(["start", dir])

	if is_mac():
		statmsg("(mac) open %s" % dir)
		subprocess.call(["open", dir])

<commit_msg>qlibutils: added find_camera() function<commit_after>"""
		@file		qlibutils.py
		@author		xy
		@since		2012-07-23

		@brief		qLib-related utility functions.

		Location: $HIH/scripts/python/

"""

import hou

import glob
import os
import platform
import re
import subprocess
import traceback

# TODO: msg functions with exception handling

def is_platform(name='none'):
	return name.lower() in platform.system().lower()

def is_linux(): return is_platform('linux')
def is_windows(): return is_platform('win')
def is_mac(): return is_platform('mac')

def statmsg(msg, warn=False):
	'''.'''
	s = hou.severityType.Warning if warn else hou.severityType.Message
	if hou.isUIAvailable():
		hou.ui.setStatusMessage(msg, severity=s)

def set_namespace_aliases(prefix="qLib::", alias=True, verbose=False):
	"""
	Defines (non-)namespaced aliases for operators with a particular namespace prefix.

	This is used for always creating the namespaced versions of assets, even if an
	older .hip file contains non-namespaced asset names.

	Mapping looks like:  <opname>  -->  <prefix>::<opname>::<version>

	@note
		IMPORTANT: Although the manual says it's fine to omit the version of a
		namespaced asset (and that would refer to the latest version),
		omitting it results in files getting messed up when loaded,
		so version numbers _are_ included in the opaliases.

	@note
		This function should be called (preferably) on Houdini startup, e.g.

		import qlibutils
		qlibutils.set_namespace_aliases( ["qLib::", "myStuff::"] )

	@todo
		For each asset, the highest version number should be found and used.
		Right now it uses the first version it founds (which is fine for now).

	"""

	if type(prefix) is list:
		for p in prefix: set_namespace_aliases(p)
		return

	assert "::" in prefix, "Include trailing '::' characters in prefix"

	cmds = []
	for file in hou.hda.loadedFiles():

		names = [ (d.nodeType().name(), d.nodeTypeCategory().name()) \
			for d in list(hou.hda.definitionsInFile(file)) \
			if prefix in d.nodeType().name() ]

		for n in names:

			try:
				# strip namespace prefix and version suffix
				old = re.sub("^[^:]+::", "", n[0])
				old = re.search("^[^:]+", old).group(0)

				# opalias <network> <namespaced-op.> <plain-old-op.>
				cmd = "opalias %s %s %s" % (n[1], n[0], old)

				if cmd not in cmds:
					if verbose: print cmd
					if alias: hou.hscript(cmd)
					cmds.append(cmd)
				else:
					print "# ALREADY ALIASED: %s (%s)" % (cmd, file)

			except:
				print "ERROR: %s" % traceback.format_exc()

def do_crash_recovery(calledFromUI=False):
	tmpdir = str(hou.getenv("TEMP"))
	files = glob.glob( os.path.join(tmpdir, '*.hip') )

	uicall = calledFromUI

	if hou.isUIAvailable() and len(files)>0:

		td = os.path.join(tmpdir, '') # dir with '/'
		files = [ (f, os.path.getmtime(f), ) for f in files ]
		files = sorted(files, key=lambda f: f[1], reverse=True)
		files = [ str(re.sub('^%s' % td, '', f[0])) for f in files ]

		sel = hou.ui.selectFromList(files, exclusive=True,
			title="Crash Recovery",
			message="Select .hip File to Recover")

		recovered = False
		if len(sel)>0:
			f = files[sel[0]]
			fn = os.path.join(tmpdir, f)

			# extract HIPNAME
			f = re.sub('^crash.', '', f)
			f = re.sub('\..+_[0-9]+\.hip', '.hip', f)

			# do recovery
			try:
				hou.hipFile.clear(True)
				hou.hipFile.load(fn, True)
				hou.setUpdateMode(hou.updateMode.Manual)
				recovered = True
			except:
				hou.ui.setStatusMessage("error while recovering file %s" % fn, hou.severityType.Error)
				print "ERROR: %s" % traceback.format_exc()

			hou.hipFile.setName(f)

		# delete crash file(s)

		msg = 'Cleanup: Delete all crash recovery hip files?'
		if recovered:
			msg = \
				'File recovered. Make sure to save it to a safe location.\n' \
				'NOTE: Update mode is set to "Manual" to avoid potential re-crashes.\n' \
				'\n%s' % msg

		d = hou.ui.displayMessage(msg, buttons=("DELETE", "Skip", ))
		if d==0:
			files = \
				glob.glob( os.path.join(tmpdir, 'crash.*') ) + \
				glob.glob( os.path.join(tmpdir, '*.hip') )
			for f in files:
			    try:
				os.remove(f)
			    except:
				pass

			hou.ui.setStatusMessage("crash recovery cleanup: deleted %d files" % len(files))
		else:
			pass # user cancelled

	else:
		# no crash files found
		#
		if uicall:
			hou.ui.setStatusMessage("  Crash Recovery:  No emergency-saved .hip file(s) found -- nothing to recover.", hou.severityType.ImportantMessage)
			pass

def open_dir(dir="", env=None):
	'''.'''
	dir=str(dir)

	if env:
		dir = str(hou.getenv(env))

	if not os.path.exists(dir):
		statmsg("Directory doesn't exist (%s)" % dir, warn=True)
		return

	if is_linux():
		statmsg("(linux) xdg-open %s" % dir)
		subprocess.call(["xdg-open", dir])

	if is_windows():
		dir = dir.replace('/', '\\')
		statmsg("(windows) start %s" % dir)
		subprocess.call(["start", dir])

	if is_mac():
		statmsg("(mac) open %s" % dir)
		subprocess.call(["open", dir])

def find_camera(oppattern, path=None):
	'''Finds a camera OBJ within nested subnets and returns its full path.'''
	r = ''
	try:
		root = hou.pwd()
		if path: root = hou.node(path)
		root = root.glob(oppattern)

		if len(root)>0:
			cam = [ n.path() for n in root[0].allSubChildren() if n.type().name()=='cam' ]
			if len(cam)>0:
				r = cam[0]
			else:
				pass # no camera found
		else:
			pass # no root node found

	except:
		pass # TODO: error handling

	return r

def backup_rop_output_file():
	'''Creates a dated backup of an output file of a ROP. Useful as a ROP Pre-Render call.'''
	pass

<|endoftext|>


<commit_before>from reference_data.models import GeneInfo

def get_genes_by_symbol_and_id():
    gene_symbol_to_genes = {}
    gene_id_to_genes = {}
    for gene_info in GeneInfo.objects.all().only('gene_id', 'gene_symbol').order_by('-gencode_release'):
        if gene_info.gene_symbol not in gene_symbol_to_genes:
            gene_symbol_to_genes[gene_info.gene_symbol] = gene_info
        if gene_info.gene_id not in gene_id_to_genes:
            gene_id_to_genes[gene_info.gene_id] = gene_info
    return gene_symbol_to_genes, gene_id_to_genes

def get_genes_by_symbol():
    gene_symbol_to_genes, _ = get_genes_by_symbol_and_id()
    return gene_symbol_to_genes

def get_genes_by_id():
    _, gene_id_to_genes = get_genes_by_symbol_and_id()
    return gene_id_to_genes
<commit_msg>remove unsused gene_util functions<commit_after>from reference_data.models import GeneInfo

def get_genes_by_symbol_and_id():
    gene_symbol_to_genes = {}
    gene_id_to_genes = {}
    for gene_info in GeneInfo.objects.all().only('gene_id', 'gene_symbol').order_by('-gencode_release'):
        if gene_info.gene_symbol not in gene_symbol_to_genes:
            gene_symbol_to_genes[gene_info.gene_symbol] = gene_info
        if gene_info.gene_id not in gene_id_to_genes:
            gene_id_to_genes[gene_info.gene_id] = gene_info
    return gene_symbol_to_genes, gene_id_to_genes
<|endoftext|>