<commit_before>#!/usr/bin/env python

# This script can be used to figure out if a job on a cluster is hung.  If all goes well, it'll print the unique
# stack traces out.

import sys, os, re, subprocess
from tempfile import TemporaryFile
from optparse import OptionParser, OptionGroup, Values

##################################################################
# Modify the following variable for your cluster
node_name_pattern = re.compile("(fission-\d{4})")
##################################################################

def generateTraces(job_num, application_name, num_hosts):
  command = "qstat -n " + job_num

  p = subprocess.Popen(command, shell=True, stdout=subprocess.PIPE)
  output = p.communicate()[0]

  # The lists of hosts
  hosts = []
  # The array of jobs
  jobs = []

  # The machine name should go here!
  host_strs = node_name_pattern.findall(output)
  for i in host_strs:
    hosts.append(i)

  # Launch all the jobs
  if num_hosts == 0:
    num_hosts = len(hosts)

  for i in range(len(hosts)):
    if i >= num_hosts:
      continue

    #command = "ssh " + host + " \"ps -e | grep " + application_name + " | awk '{print \$1}' | xargs -I {} gdb --batch --pid={} -ex bt 2>&1 | grep '^#' \""
    command = "ssh " + hosts[i] + " \"ps -e | grep " + application_name + " | awk '{print \$1}' | xargs -I '{}' sh -c 'echo Host: " + hosts[i] + " PID: {}; pstack {}; printf '*%.0s' {1..80}; echo' \""
    f = TemporaryFile()
    p = subprocess.Popen(command, stdout=f, close_fds=False, shell=True)
    jobs.append((p, f))

  # Now process the output from each of the jobs
  traces = []
  for (p, f) in jobs:
    p.wait()
    f.seek(0)
    output = f.read()
    f.close()

    # strip blank lines
    output = os.linesep.join([s for s in output.splitlines() if s])

    traces.extend(splitTraces(output))

  return traces

def readTracesFromFile(filename):
  f = open(filename)
  data = f.read()
  return splitTraces(data)

def splitTraces(trace_string):
  trace_regex = re.compile("^\**\n", re.M)
  traces = trace_regex.split(trace_string)

#  # Only keep lines beginning with a #
#  throw_away = re.compile("^[^#].*", re.M)
#  traces = [throw_away.sub("", trace) for trace in traces]

  return traces

# Process the individual traces
def processTraces(traces, num_lines_to_keep):
  unique_stack_traces = {}
  last_lines_regex = re.compile("(?:.*\n){" + str(num_lines_to_keep) + "}\Z", re.M)
  host_regex = re.compile("^(Host.*)", re.M)

  for trace in traces:
    if len(trace) == 0:
      continue

    # Grab the host and PID
    m = host_regex.search(trace)
    if m:
      host_pid = m.group(1)

    # If the user requested to save only the last few lines, do that here
    if num_lines_to_keep:
      m = last_lines_regex.search(trace)
      if m:
        trace = m.group(0)

    unique = ''
    for bt in unique_stack_traces:
      if compareTraces(trace, bt):
        unique = bt

    if unique == '':
      unique_stack_traces[trace] = [host_pid]
    else:
      unique_stack_traces[unique].append(host_pid)

  return unique_stack_traces

def compareTraces(trace1, trace2):
  lines1 = trace1.split("\n")
  lines2 = trace2.split("\n")

  if len(lines1) != len(lines2):
    return False

  # Only compare the stack trace part - not the memory addresses
  # Note this subroutine may need tweaking if the stack trace is different
  # on the current machine
  memory_re = re.compile("0x[0-9a-f]*")
  for i in xrange(len(lines1)):
    line1 = lines1[i].split()[2:]
    line2 = lines2[i].split()[2:]

    # Let's strip out all the memory addresses too
    line1 = [memory_re.sub("0x...", line) for line in line1]
    line2 = [memory_re.sub("0x...", line) for line in line2]

    if line1 != line2:
      return False

  return True

def main():
  parser = OptionParser(usage='Usage: %prog [options] <PBS Job num> <Application>')
  parser.add_option('-s', '--stacks', action='store', dest='stacks', type='int', default=0, help="The number of stack frames to keep and compare for uniqueness (Default: ALL)")
  parser.add_option('-n', '--hosts', action='store', dest='hosts', type='int', default=0, help="The number of hosts to visit (Default: ALL)")
  parser.add_option('-f', '--force', action='store_true', dest='force', default=False, help="Whether or not to force a regen if a cache file exists")
  (options, args) = parser.parse_args()

  if len(args) != 2:
    parser.print_help()
    sys.exit(1)

  # The PBS job number and the application should be passed on the command line
  # Additionally, an optional argument of the number of frames to keep (compare) may be passed
  job_num = args[0]
  application = args[1]
  num_to_keep = options.stacks
  num_hosts = options.hosts

  # first see if there is a cache file available
  cache_filename = application + '.' + job_num + '.cache'

  traces = []
  if not os.path.exists(cache_filename) or options.force:
    traces = generateTraces(job_num, application, options.hosts)

    # Cache the restuls to a file
    cache_file = open(cache_filename, 'w')
    for trace in traces:
      cache_file.write(trace + "*"*80 + "\n")
    cache_file.write("\n")
    cache_file.close()

  # Process the traces to collapse them into unique stacks
  traces = readTracesFromFile(cache_filename)
  unique_stack_traces = processTraces(traces, num_to_keep)

  print "Unique Stack Traces"
  for trace, count in unique_stack_traces.iteritems():
    print "*"*80 + "\nCount: " + str(len(count)) + "\n"
    if len(count) < 10:
      print "\n".join(count)
    print "\n" + trace

if __name__ == '__main__':
  main()
<commit_msg>Update find_hung_process.py to work with Falcon or Bechler clusters closes #4216<commit_after>#!/usr/bin/env python

# This script can be used to figure out if a job on a cluster is hung.  If all goes well, it'll print the unique
# stack traces out.

import sys, os, re, subprocess
from tempfile import TemporaryFile
from optparse import OptionParser, OptionGroup, Values

##################################################################
# Modify the following variable(s) for your cluster or use one of the versions below
### FISSION
#node_name_pattern = re.compile("(fission-\d{4})")
#pstack_binary = 'pstack'
### BECHLER
#node_name_pattern = re.compile("(b\d{2})")
#pstack_binary = 'pstack'
### FALCON
node_name_pattern = re.compile("(r\di\dn\d{1,2})")
pstack_binary = 'gstack'
##################################################################

def generateTraces(job_num, application_name, num_hosts):
  command = "qstat -n " + job_num

  p = subprocess.Popen(command, shell=True, stdout=subprocess.PIPE)
  output = p.communicate()[0]

  # The lists of hosts
  hosts = []
  # The array of jobs
  jobs = []

  # The machine name should go here!
  host_strs = node_name_pattern.findall(output)
  for i in host_strs:
    hosts.append(i)

  # Launch all the jobs
  if num_hosts == 0:
    num_hosts = len(hosts)

  for i in range(len(hosts)):
    if i >= num_hosts:
      continue

    #command = "ssh " + host + " \"ps -e | grep " + application_name + " | awk '{print \$1}' | xargs -I {} gdb --batch --pid={} -ex bt 2>&1 | grep '^#' \""
    command = "ssh " + hosts[i] + " \"ps -e | grep " + application_name + " | awk '{print \$1}' | xargs -I '{}' sh -c 'echo Host: " + hosts[i] + " PID: {}; " + pstack_binary + " {}; printf '*%.0s' {1..80}; echo' \""
    f = TemporaryFile()
    p = subprocess.Popen(command, stdout=f, close_fds=False, shell=True)
    jobs.append((p, f))

  # Now process the output from each of the jobs
  traces = []
  for (p, f) in jobs:
    p.wait()
    f.seek(0)
    output = f.read()
    f.close()

    # strip blank lines
    output = os.linesep.join([s for s in output.splitlines() if s])

    traces.extend(splitTraces(output))

  return traces

def readTracesFromFile(filename):
  f = open(filename)
  data = f.read()
  return splitTraces(data)

def splitTraces(trace_string):
  trace_regex = re.compile("^\**\n", re.M)
  traces = trace_regex.split(trace_string)

#  # Only keep lines beginning with a #
#  throw_away = re.compile("^[^#].*", re.M)
#  traces = [throw_away.sub("", trace) for trace in traces]

  return traces

# Process the individual traces
def processTraces(traces, num_lines_to_keep):
  unique_stack_traces = {}
  last_lines_regex = re.compile("(?:.*\n){" + str(num_lines_to_keep) + "}\Z", re.M)
  host_regex = re.compile("^(Host.*)", re.M)

  for trace in traces:
    if len(trace) == 0:
      continue

    # Grab the host and PID
    m = host_regex.search(trace)
    if m:
      host_pid = m.group(1)

    # If the user requested to save only the last few lines, do that here
    if num_lines_to_keep:
      m = last_lines_regex.search(trace)
      if m:
        trace = m.group(0)

    unique = ''
    for bt in unique_stack_traces:
      if compareTraces(trace, bt):
        unique = bt

    if unique == '':
      unique_stack_traces[trace] = [host_pid]
    else:
      unique_stack_traces[unique].append(host_pid)

  return unique_stack_traces

def compareTraces(trace1, trace2):
  lines1 = trace1.split("\n")
  lines2 = trace2.split("\n")

  if len(lines1) != len(lines2):
    return False

  # Only compare the stack trace part - not the memory addresses
  # Note this subroutine may need tweaking if the stack trace is different
  # on the current machine
  memory_re = re.compile("0x[0-9a-f]*")
  for i in xrange(len(lines1)):
    line1 = lines1[i].split()[2:]
    line2 = lines2[i].split()[2:]

    # Let's strip out all the memory addresses too
    line1 = [memory_re.sub("0x...", line) for line in line1]
    line2 = [memory_re.sub("0x...", line) for line in line2]

    if line1 != line2:
      return False

  return True

def main():
  parser = OptionParser(usage='Usage: %prog [options] <PBS Job num> <Application>')
  parser.add_option('-s', '--stacks', action='store', dest='stacks', type='int', default=0, help="The number of stack frames to keep and compare for uniqueness (Default: ALL)")
  parser.add_option('-n', '--hosts', action='store', dest='hosts', type='int', default=0, help="The number of hosts to visit (Default: ALL)")
  parser.add_option('-f', '--force', action='store_true', dest='force', default=False, help="Whether or not to force a regen if a cache file exists")
  (options, args) = parser.parse_args()

  if len(args) != 2:
    parser.print_help()
    sys.exit(1)

  # The PBS job number and the application should be passed on the command line
  # Additionally, an optional argument of the number of frames to keep (compare) may be passed
  job_num = args[0]
  application = args[1]
  num_to_keep = options.stacks
  num_hosts = options.hosts

  # first see if there is a cache file available
  cache_filename = application + '.' + job_num + '.cache'

  traces = []
  if not os.path.exists(cache_filename) or options.force:
    traces = generateTraces(job_num, application, options.hosts)

    # Cache the restuls to a file
    cache_file = open(cache_filename, 'w')
    for trace in traces:
      cache_file.write(trace + "*"*80 + "\n")
    cache_file.write("\n")
    cache_file.close()

  # Process the traces to collapse them into unique stacks
  traces = readTracesFromFile(cache_filename)
  unique_stack_traces = processTraces(traces, num_to_keep)

  print "Unique Stack Traces"
  for trace, count in unique_stack_traces.iteritems():
    print "*"*80 + "\nCount: " + str(len(count)) + "\n"
    if len(count) < 10:
      print "\n".join(count)
    print "\n" + trace

if __name__ == '__main__':
  main()
<|endoftext|>


<commit_before>'''
Created on Oct 5, 2010

@author: Mark V Systems Limited
(c) Copyright 2010 Mark V Systems Limited, All rights reserved.
'''

# initialize object from loaded linkbases
from collections import defaultdict
from arelle import (ModelDtsObject, XbrlConst, XmlUtil, ModelValue)
from arelle.ModelObject import ModelObject
from arelle.ModelDtsObject import ModelResource
import os

def create(modelXbrl, arcrole, linkrole=None, linkqname=None, arcqname=None, includeProhibits=False):
    return ModelRelationshipSet(modelXbrl, arcrole, linkrole, linkqname, arcqname, includeProhibits)

def ineffectiveArcs(baseSetModelLinks, arcrole, arcqname=None):
    relationships = defaultdict(list)
    for modelLink in baseSetModelLinks:
        for linkChild in modelLink.getchildren():
            if (isinstance(linkChild,ModelObject) and
                linkChild.get("{http://www.w3.org/1999/xlink}type") == "arc" and
                arcrole == linkChild.get("{http://www.w3.org/1999/xlink}arcrole") and
                (arcqname is None or arcqname == linkChild)):
                fromLabel = linkChild.get("{http://www.w3.org/1999/xlink}from")
                toLabel = linkChild.get("{http://www.w3.org/1999/xlink}to")
                for fromResource in modelLink.labeledResources[fromLabel]:
                    for toResource in modelLink.labeledResources[toLabel]:
                        modelRel = ModelDtsObject.ModelRelationship(modelLink.modelDocument, linkChild, fromResource.dereference(), toResource.dereference())
                        relationships[modelRel.equivalenceKey].append(modelRel)
    # determine ineffective relationships
    ineffectives = []
    for equivalenceKey, relationship in relationships.items():
        #sort by priority, prohibited
        equivalentRels = []
        for i, modelRel in enumerate(relationship):
            equivalentRels.append((modelRel.priority,modelRel.prohibitedUseSortKey,i))
        equivalentRels.sort()
        priorRel = None
        for rel in equivalentRels:
            if rel[1] == 2: # this rel is prohibited
                if priorRel is None:
                    ineffective = relationship[rel[2]]
                    ineffective.ineffectivity = _("prohibited arc (priority {0}) has no other arc to prohibit").format(
                                               ineffective.priority)
                    ineffectives.append(ineffective) # this rel ineffective
                elif priorRel[1] == 2: # prior rel is prohibited
                    ineffective = relationship[priorRel[2]]
                    effective = relationship[rel[2]]
                    ineffective.ineffectivity = _("prohibited arc (priority {0}, {1} - {2}) has an equivalent prohibited arc (priority {3}, {4} - {5})\n").format(
                                             ineffective.priority, ineffective.modelDocument.basename, ineffective.sourceline,
                                             effective.priority, effective.modelDocument.basename, effective.sourceline)
                    ineffectives.append(ineffective)
            else:
                if priorRel is not None and priorRel[1] != 2:
                    ineffective = relationship[priorRel[2]]
                    effective = relationship[rel[2]]
                    ineffective.ineffectivity = _("arc (priority {0}, {1} - {2}) is ineffective due to equivalent arc (priority {3}, {4} - {5})\n").format(
                                             ineffective.priority, ineffective.modelDocument.basename, ineffective.sourceline,
                                             effective.priority, effective.modelDocument.basename, effective.sourceline)
                    ineffectives.append(ineffective) # prior ineffective
            priorRel = rel
    return ineffectives

def baseSetArcroles(modelXbrl):
    # returns sorted list of tuples of arcrole basename and uri
    return sorted(set((XbrlConst.baseSetArcroleLabel(b[0]),b[0]) for b in modelXbrl.baseSets.keys()))

def labelroles(modelXbrl, includeConceptName=False):
    # returns sorted list of tuples of arcrole basename and uri
    return sorted(set((XbrlConst.labelroleLabel(r),r)
                        for r in (modelXbrl.labelroles | ({XbrlConst.conceptNameLabelRole} if includeConceptName else set()))
                        if r is not None))

class ModelRelationshipSet:
    def __init__(self, modelXbrl, arcrole, linkrole=None, linkqname=None, arcqname=None, includeProhibits=False):
        self.isChanged = False
        self.modelXbrl = modelXbrl
        self.arcrole = arcrole
        self.linkrole = linkrole
        self.linkqname = linkqname
        self.arcqname = arcqname

        baseSetKey = (arcrole, linkrole, linkqname, arcqname)
        relationshipSetKey = (arcrole, linkrole, linkqname, arcqname, includeProhibits)

        # base sets does not care about the #includeProhibits
        if baseSetKey in self.modelXbrl.baseSets:
            modelLinks = self.modelXbrl.baseSets[baseSetKey]
        else:
            modelLinks = []

        # gather arcs
        relationships = {}
        isDimensionRel =  self.arcrole == "XBRL-dimensions" # all dimensional relationship arcroles
        isFormulaRel =  self.arcrole == "XBRL-formulae" # all formula relationship arcroles
        isTableRenderingRel = self.arcrole == "Table-rendering"
        isFootnoteRel =  self.arcrole == "XBRL-footnotes" # all footnote relationship arcroles

        for modelLink in modelLinks:
            arcs = []
            linkEltQname = modelLink.qname
            for linkChild in modelLink.getchildren():
                linkChildArcrole = linkChild.get("{http://www.w3.org/1999/xlink}arcrole")
                if linkChild.get("{http://www.w3.org/1999/xlink}type") == "arc" and linkChildArcrole:
                    linkChildQname = linkChild
                    if isFootnoteRel:
                        arcs.append(linkChild)
                    elif isDimensionRel:
                        if XbrlConst.isDimensionArcrole(linkChildArcrole):
                            arcs.append(linkChild)
                    elif isFormulaRel:
                        if XbrlConst.isFormulaArcrole(linkChildArcrole):
                            arcs.append(linkChild)
                    elif isTableRenderingRel:
                        if XbrlConst.isTableRenderingArcrole(linkChildArcrole):
                            arcs.append(linkChild)
                    elif arcrole == linkChildArcrole and \
                         (arcqname is None or arcqname == linkChildQname) and \
                         (linkqname is None or linkqname == linkEltQname):
                        arcs.append(linkChild)

            # build network
            for arcElement in arcs:
                arcrole = arcElement.get("{http://www.w3.org/1999/xlink}arcrole")
                fromLabel = arcElement.get("{http://www.w3.org/1999/xlink}from")
                toLabel = arcElement.get("{http://www.w3.org/1999/xlink}to")
                for fromResource in modelLink.labeledResources[fromLabel]:
                    for toResource in modelLink.labeledResources[toLabel]:
                        if isinstance(fromResource,ModelResource) and isinstance(toResource,ModelResource):
                            modelRel = ModelDtsObject.ModelRelationship(modelLink.modelDocument, arcElement, fromResource.dereference(), toResource.dereference())
                            modelRelEquivalenceKey = modelRel.equivalenceKey    # this is a complex tuple to compute, get once for below
                            if modelRelEquivalenceKey not in relationships or \
                               modelRel.priorityOver(relationships[modelRelEquivalenceKey]):
                                relationships[modelRelEquivalenceKey] = modelRel

        #reduce effective arcs and order relationships...
        self.modelRelationships = []
        self.modelRelationshipsFrom = None
        self.modelRelationshipsTo = None
        self.modelConceptRoots = None
        self.modellinkRoleUris = None
        orderRels = defaultdict(list)
        for modelRel in relationships.values():
            if includeProhibits or not modelRel.isProhibited:
                orderRels[modelRel.order].append(modelRel)
        for order in sorted(orderRels.keys()):
            for modelRel in orderRels[order]:
                self.modelRelationships.append(modelRel)
        modelXbrl.relationshipSets[relationshipSetKey] = self

    @property
    def linkRoleUris(self):
        if self.modellinkRoleUris is None:
            self.modellinkRoleUris = set(modelRel.linkrole for modelRel in self.modelRelationships)
        return self.modellinkRoleUris

    def loadModelRelationshipsFrom(self):
        if self.modelRelationshipsFrom is None:
            self.modelRelationshipsFrom = defaultdict(list)
            for modelRel in self.modelRelationships:
                fromModelObject = modelRel.fromModelObject
                if fromModelObject is not None: # none if concepts failed to load
                    self.modelRelationshipsFrom[fromModelObject].append(modelRel)

    def loadModelRelationshipsTo(self):
        if self.modelRelationshipsTo is None:
            self.modelRelationshipsTo = defaultdict(list)
            for modelRel in self.modelRelationships:
                toModelObject = modelRel.toModelObject
                if toModelObject is not None:   # none if concepts failed to load
                    self.modelRelationshipsTo[toModelObject].append(modelRel)

    def fromModelObjects(self):
        self.loadModelRelationshipsFrom()
        return self.modelRelationshipsFrom

    def fromModelObject(self, modelFrom):
        self.loadModelRelationshipsFrom()
        return self.modelRelationshipsFrom.get(modelFrom, [])

    def toModelObjects(self):
        self.loadModelRelationshipsTo()
        return self.modelRelationshipsTo

    def toModelObject(self, modelTo):
        self.loadModelRelationshipsTo()
        return self.modelRelationshipsTo.get(modelTo, [])

    @property
    def rootConcepts(self):
        if self.modelConceptRoots is None:
            self.loadModelRelationshipsFrom()
            self.loadModelRelationshipsTo()
            self.modelConceptRoots = []
            for modelRelFrom in self.modelRelationshipsFrom.keys():
                if self.modelRelationshipsTo.get(modelRelFrom) == None and \
                    modelRelFrom not in self.modelConceptRoots:
                    self.modelConceptRoots.append(modelRelFrom)
        return self.modelConceptRoots

    # if modelFrom and modelTo are provided determine that they have specified relationship
    # if only modelFrom, determine that there are relationships present of specified axis
    def isRelated(self, modelFrom, axis, modelTo=None, visited=None): # either model concept or qname
        if visited is None: visited = set()
        if isinstance(modelFrom,ModelValue.QName): modelFrom = self.modelXbrl.qnameConcepts[modelFrom]
        if isinstance(modelTo,ModelValue.QName): modelTo = self.modelXbrl.qnameConcepts[modelTo]
        if axis.endswith("self") and (modelTo is None or modelFrom == modelTo):
            return True
        for modelRel in self.fromModelObject(modelFrom):
            toConcept = modelRel.toModelObject
            if modelTo is None or modelTo == toConcept:
                return True
            if axis.startswith("descendant") and toConcept not in visited:
                visited.add(toConcept)
                if self.isRelated(toConcept, axis, modelTo, visited):
                    return True
                visited.discard(toConcept)
        return False

    def label(self, modelFrom, role, lang, returnMultiple=False, returnText=True):
        shorterLangInLabel = longerLangInLabel = None
        shorterLangLabels = longerLangLabels = None
        langLabels = []
        for modelLabelRel in self.fromModelObject(modelFrom):
            label = modelLabelRel.toModelObject
            if role == label.role:
                labelLang = label.xmlLang
                text = label.text if returnText else label
                if lang is None or len(lang) == 0 or lang == labelLang:
                    langLabels.append(text)
                    if not returnMultiple:
                        break
                elif labelLang.startswith(lang):
                    if not longerLangInLabel or len(longerLangInLabel) > len(labelLang):
                        longerLangInLabel = labelLang
                        longerLangLabels = [text,]
                    else:
                        longerLangLabels.append(text)
                elif lang.startswith(labelLang):
                    if not shorterLangInLabel or len(shorterLangInLabel) < len(labelLang):
                        shorterLangInLabel = labelLang
                        shorterLangLabels = [text,]
                    else:
                        shorterLangLabels.append(text)
        if langLabels:
            if returnMultiple: return langLabels
            else: return langLabels[0]
        if shorterLangLabels:  # more general has preference
            if returnMultiple: return shorterLangLabels
            else: return shorterLangLabels[0]
        if longerLangLabels:
            if returnMultiple: return longerLangLabels
            else: return longerLangLabels[0]
        return None
<commit_msg>simplify ineffective arcs code expression (used in EFM/GFM validation)<commit_after>'''
Created on Oct 5, 2010

@author: Mark V Systems Limited
(c) Copyright 2010 Mark V Systems Limited, All rights reserved.
'''

# initialize object from loaded linkbases
from collections import defaultdict
from arelle import (ModelDtsObject, XbrlConst, XmlUtil, ModelValue)
from arelle.ModelObject import ModelObject
from arelle.ModelDtsObject import ModelResource
import os

def create(modelXbrl, arcrole, linkrole=None, linkqname=None, arcqname=None, includeProhibits=False):
    return ModelRelationshipSet(modelXbrl, arcrole, linkrole, linkqname, arcqname, includeProhibits)

def ineffectiveArcs(baseSetModelLinks, arcrole, arcqname=None):
    relationships = defaultdict(list)
    for modelLink in baseSetModelLinks:
        for linkChild in modelLink.getchildren():
            if (isinstance(linkChild,ModelObject) and
                linkChild.get("{http://www.w3.org/1999/xlink}type") == "arc" and
                arcrole == linkChild.get("{http://www.w3.org/1999/xlink}arcrole") and
                (arcqname is None or arcqname == linkChild)):
                fromLabel = linkChild.get("{http://www.w3.org/1999/xlink}from")
                toLabel = linkChild.get("{http://www.w3.org/1999/xlink}to")
                for fromResource in modelLink.labeledResources[fromLabel]:
                    for toResource in modelLink.labeledResources[toLabel]:
                        modelRel = ModelDtsObject.ModelRelationship(modelLink.modelDocument, linkChild, fromResource.dereference(), toResource.dereference())
                        relationships[modelRel.equivalenceKey].append(modelRel)
    # determine ineffective relationships
    ineffectives = []
    for equivalenceKey, relationship in relationships.items():
        #sort by priority, prohibited
        equivalentRels = [(modelRel.priority, modelRel.prohibitedUseSortKey, i)
                          for i, modelRel in enumerate(relationship)]
        priorRel = None
        for rel in sorted( equivalentRels ):
            if rel[1] == 2: # this rel is prohibited
                if priorRel is None:
                    ineffective = relationship[rel[2]]
                    ineffective.ineffectivity = _("prohibited arc (priority {0}) has no other arc to prohibit").format(
                                               ineffective.priority)
                    ineffectives.append(ineffective) # this rel ineffective
                elif priorRel[1] == 2: # prior rel is prohibited
                    ineffective = relationship[priorRel[2]]
                    effective = relationship[rel[2]]
                    ineffective.ineffectivity = _("prohibited arc (priority {0}, {1} - {2}) has an equivalent prohibited arc (priority {3}, {4} - {5})\n").format(
                                             ineffective.priority, ineffective.modelDocument.basename, ineffective.sourceline,
                                             effective.priority, effective.modelDocument.basename, effective.sourceline)
                    ineffectives.append(ineffective)
            else:
                if priorRel is not None and priorRel[1] != 2:
                    ineffective = relationship[priorRel[2]]
                    effective = relationship[rel[2]]
                    ineffective.ineffectivity = _("arc (priority {0}, {1} - {2}) is ineffective due to equivalent arc (priority {3}, {4} - {5})\n").format(
                                             ineffective.priority, ineffective.modelDocument.basename, ineffective.sourceline,
                                             effective.priority, effective.modelDocument.basename, effective.sourceline)
                    ineffectives.append(ineffective) # prior ineffective
            priorRel = rel
    return ineffectives

def baseSetArcroles(modelXbrl):
    # returns sorted list of tuples of arcrole basename and uri
    return sorted(set((XbrlConst.baseSetArcroleLabel(b[0]),b[0]) for b in modelXbrl.baseSets.keys()))

def labelroles(modelXbrl, includeConceptName=False):
    # returns sorted list of tuples of arcrole basename and uri
    return sorted(set((XbrlConst.labelroleLabel(r),r)
                        for r in (modelXbrl.labelroles | ({XbrlConst.conceptNameLabelRole} if includeConceptName else set()))
                        if r is not None))

class ModelRelationshipSet:
    def __init__(self, modelXbrl, arcrole, linkrole=None, linkqname=None, arcqname=None, includeProhibits=False):
        self.isChanged = False
        self.modelXbrl = modelXbrl
        self.arcrole = arcrole
        self.linkrole = linkrole
        self.linkqname = linkqname
        self.arcqname = arcqname

        baseSetKey = (arcrole, linkrole, linkqname, arcqname)
        relationshipSetKey = (arcrole, linkrole, linkqname, arcqname, includeProhibits)

        # base sets does not care about the #includeProhibits
        if baseSetKey in self.modelXbrl.baseSets:
            modelLinks = self.modelXbrl.baseSets[baseSetKey]
        else:
            modelLinks = []

        # gather arcs
        relationships = {}
        isDimensionRel =  self.arcrole == "XBRL-dimensions" # all dimensional relationship arcroles
        isFormulaRel =  self.arcrole == "XBRL-formulae" # all formula relationship arcroles
        isTableRenderingRel = self.arcrole == "Table-rendering"
        isFootnoteRel =  self.arcrole == "XBRL-footnotes" # all footnote relationship arcroles

        for modelLink in modelLinks:
            arcs = []
            linkEltQname = modelLink.qname
            for linkChild in modelLink.getchildren():
                linkChildArcrole = linkChild.get("{http://www.w3.org/1999/xlink}arcrole")
                if linkChild.get("{http://www.w3.org/1999/xlink}type") == "arc" and linkChildArcrole:
                    linkChildQname = linkChild
                    if isFootnoteRel:
                        arcs.append(linkChild)
                    elif isDimensionRel:
                        if XbrlConst.isDimensionArcrole(linkChildArcrole):
                            arcs.append(linkChild)
                    elif isFormulaRel:
                        if XbrlConst.isFormulaArcrole(linkChildArcrole):
                            arcs.append(linkChild)
                    elif isTableRenderingRel:
                        if XbrlConst.isTableRenderingArcrole(linkChildArcrole):
                            arcs.append(linkChild)
                    elif arcrole == linkChildArcrole and \
                         (arcqname is None or arcqname == linkChildQname) and \
                         (linkqname is None or linkqname == linkEltQname):
                        arcs.append(linkChild)

            # build network
            for arcElement in arcs:
                arcrole = arcElement.get("{http://www.w3.org/1999/xlink}arcrole")
                fromLabel = arcElement.get("{http://www.w3.org/1999/xlink}from")
                toLabel = arcElement.get("{http://www.w3.org/1999/xlink}to")
                for fromResource in modelLink.labeledResources[fromLabel]:
                    for toResource in modelLink.labeledResources[toLabel]:
                        if isinstance(fromResource,ModelResource) and isinstance(toResource,ModelResource):
                            modelRel = ModelDtsObject.ModelRelationship(modelLink.modelDocument, arcElement, fromResource.dereference(), toResource.dereference())
                            modelRelEquivalenceKey = modelRel.equivalenceKey    # this is a complex tuple to compute, get once for below
                            if modelRelEquivalenceKey not in relationships or \
                               modelRel.priorityOver(relationships[modelRelEquivalenceKey]):
                                relationships[modelRelEquivalenceKey] = modelRel

        #reduce effective arcs and order relationships...
        self.modelRelationships = []
        self.modelRelationshipsFrom = None
        self.modelRelationshipsTo = None
        self.modelConceptRoots = None
        self.modellinkRoleUris = None
        orderRels = defaultdict(list)
        for modelRel in relationships.values():
            if includeProhibits or not modelRel.isProhibited:
                orderRels[modelRel.order].append(modelRel)
        for order in sorted(orderRels.keys()):
            for modelRel in orderRels[order]:
                self.modelRelationships.append(modelRel)
        modelXbrl.relationshipSets[relationshipSetKey] = self

    @property
    def linkRoleUris(self):
        if self.modellinkRoleUris is None:
            self.modellinkRoleUris = set(modelRel.linkrole for modelRel in self.modelRelationships)
        return self.modellinkRoleUris

    def loadModelRelationshipsFrom(self):
        if self.modelRelationshipsFrom is None:
            self.modelRelationshipsFrom = defaultdict(list)
            for modelRel in self.modelRelationships:
                fromModelObject = modelRel.fromModelObject
                if fromModelObject is not None: # none if concepts failed to load
                    self.modelRelationshipsFrom[fromModelObject].append(modelRel)

    def loadModelRelationshipsTo(self):
        if self.modelRelationshipsTo is None:
            self.modelRelationshipsTo = defaultdict(list)
            for modelRel in self.modelRelationships:
                toModelObject = modelRel.toModelObject
                if toModelObject is not None:   # none if concepts failed to load
                    self.modelRelationshipsTo[toModelObject].append(modelRel)

    def fromModelObjects(self):
        self.loadModelRelationshipsFrom()
        return self.modelRelationshipsFrom

    def fromModelObject(self, modelFrom):
        self.loadModelRelationshipsFrom()
        return self.modelRelationshipsFrom.get(modelFrom, [])

    def toModelObjects(self):
        self.loadModelRelationshipsTo()
        return self.modelRelationshipsTo

    def toModelObject(self, modelTo):
        self.loadModelRelationshipsTo()
        return self.modelRelationshipsTo.get(modelTo, [])

    @property
    def rootConcepts(self):
        if self.modelConceptRoots is None:
            self.loadModelRelationshipsFrom()
            self.loadModelRelationshipsTo()
            self.modelConceptRoots = []
            for modelRelFrom in self.modelRelationshipsFrom.keys():
                if self.modelRelationshipsTo.get(modelRelFrom) == None and \
                    modelRelFrom not in self.modelConceptRoots:
                    self.modelConceptRoots.append(modelRelFrom)
        return self.modelConceptRoots

    # if modelFrom and modelTo are provided determine that they have specified relationship
    # if only modelFrom, determine that there are relationships present of specified axis
    def isRelated(self, modelFrom, axis, modelTo=None, visited=None): # either model concept or qname
        if visited is None: visited = set()
        if isinstance(modelFrom,ModelValue.QName): modelFrom = self.modelXbrl.qnameConcepts[modelFrom]
        if isinstance(modelTo,ModelValue.QName): modelTo = self.modelXbrl.qnameConcepts[modelTo]
        if axis.endswith("self") and (modelTo is None or modelFrom == modelTo):
            return True
        for modelRel in self.fromModelObject(modelFrom):
            toConcept = modelRel.toModelObject
            if modelTo is None or modelTo == toConcept:
                return True
            if axis.startswith("descendant") and toConcept not in visited:
                visited.add(toConcept)
                if self.isRelated(toConcept, axis, modelTo, visited):
                    return True
                visited.discard(toConcept)
        return False

    def label(self, modelFrom, role, lang, returnMultiple=False, returnText=True):
        shorterLangInLabel = longerLangInLabel = None
        shorterLangLabels = longerLangLabels = None
        langLabels = []
        for modelLabelRel in self.fromModelObject(modelFrom):
            label = modelLabelRel.toModelObject
            if role == label.role:
                labelLang = label.xmlLang
                text = label.text if returnText else label
                if lang is None or len(lang) == 0 or lang == labelLang:
                    langLabels.append(text)
                    if not returnMultiple:
                        break
                elif labelLang.startswith(lang):
                    if not longerLangInLabel or len(longerLangInLabel) > len(labelLang):
                        longerLangInLabel = labelLang
                        longerLangLabels = [text,]
                    else:
                        longerLangLabels.append(text)
                elif lang.startswith(labelLang):
                    if not shorterLangInLabel or len(shorterLangInLabel) < len(labelLang):
                        shorterLangInLabel = labelLang
                        shorterLangLabels = [text,]
                    else:
                        shorterLangLabels.append(text)
        if langLabels:
            if returnMultiple: return langLabels
            else: return langLabels[0]
        if shorterLangLabels:  # more general has preference
            if returnMultiple: return shorterLangLabels
            else: return shorterLangLabels[0]
        if longerLangLabels:
            if returnMultiple: return longerLangLabels
            else: return longerLangLabels[0]
        return None
<|endoftext|>


<commit_before>import nltk
from nltk.corpus import cmudict
import curses
from curses.ascii import isdigit
import re
from nltk.probability import FreqDist
from app import db
import models
from random import randrange

d = cmudict.dict()

def numSylsInWord(word):
	if word.lower() in d:
		return [len(list(y for y in x if y[-1].isdigit())) for x in d[word.lower()]][0]

def isHaiku(potentialHaiku):
	syllableCount = countSyllables(potentialHaiku)
	if syllableCount == 17:
		#how do we want to break this down?
		#what are the other "ifs" that this conditional needs to meet?
		result = True
	else:
		result = False
	return result

def countSyllables(potentialHaiku):
	stripPunctuation = re.sub(ur"[^\w\d'\s]+",' ',potentialHaiku)
	wordsInHaiku = stripPunctuation.split()
	syllableCount = 0
	for i in wordsInHaiku:
		syllableCount += numSylsInWord(i)
	return syllableCount

def inDatabase(firstWord):
	container = []
	from models import Unigram
	unigrams = Unigram.query.filter(Unigram.word1 == firstWord)
	for each in unigrams:
			for unigram in range(each.count):
					container.append(each.word2)
	if not container:
		return False
	if container:
		return True

def generateHaiku(firstWord):
	inDB = inDatabase(firstWord)
	if inDB:
		haiku = ""
		haiku += startGenerateLine(5, firstWord)
		haiku += "\n"
		haiku += startGenerateLine(7)
		haiku += "\n"
		haiku += startGenerateLine(5)
	if not inDB:
		firstWord = pickRandomWord(5)
		haiku = generateHaiku(firstWord)
	return haiku

def startGenerateLine(sylCount, startingWord= None):
	if not startingWord:
		startingWord = pickRandomWord(sylCount)
	remainingSylCount = sylCount - countSyllables(startingWord)
	line = buildLineList(remainingSylCount, [startingWord])
	return " ".join(line)

def buildLineList(sylCount, wordsFromBefore):
	if sylCount == 0:
		return wordsFromBefore
	lastWord = wordsFromBefore[-1]
	possibilities = grabPossibleWords(lastWord, sylCount)
	for possibleWord in possibilities:
		newWordsFromBefore = [word[:] for word in wordsFromBefore]
		newWordsFromBefore.append(possibleWord)
		newSyllableCount = sylCount - countSyllables(possibleWord)
		result = buildLineList(newSyllableCount, newWordsFromBefore)
		if result:
			return result
	return None

def pickRandomWord(reqSylCount):
	from models import Unigram
	lengthDB = Unigram.query.count()
	while True:
		randomNumPick = randrange(1, lengthDB)
		tryWord = Unigram.query.filter(Unigram.id == randomNumPick).first()
		if countSyllables(tryWord.word1) <= reqSylCount:
			word = tryWord.word1
			break
	return tryWord.word1

def formatPossibleWords(unigrams, reqSylCount):
	tempContainer = []
	container = []
	if reqSylCount == 1 or reqSylCount == 2:
		for each in unigrams:
			tempContainer.append(each)
		newTempContainer = removePartOfSpeech('IN', tempContainer)
		nextTempContainer = removePartOfSpeech('CC', newTempContainer)
		finalTempContainer = removePartOfSpeech('DT', nextTempContainer)
		for unigram in finalTempContainer:
			for index in range(unigram.count):
				if countSyllables(unigram.word2) <= reqSylCount:
					container.append(unigram.word2)
	else:
		for each in unigrams:
			for index in range(each.count):
				if countSyllables(each.word2) <= reqSylCount:
					container.append(each.word2)
	return container

def removePartOfSpeech(pos, tempContainer):
	for unigram in tempContainer:
		if identifyPartsOfSpeech(unigram.word2) == pos:
			tempContainer.remove(unigram)
	return tempContainer

def grabPossibleWords(baseWord, reqSylCount):
	from models import Unigram
	listOfUnigrams = Unigram.query.filter(Unigram.word1 ==baseWord)
	return formatPossibleWords(listOfUnigrams, reqSylCount)

def identifyPartsOfSpeech(word):
	cleanString = re.sub(ur"[^\w\d'\s]+",' ', word)
	pos = nltk.word_tokenize(cleanString)
	result = nltk.pos_tag(pos)
	return result[0][1]

# def findFrequency(largeBodyofText):
#   cleanText = re.sub(ur"[^\w\d'\s]+",' ', largeBodyofText)
#   arrayOfWords = nltk.word_tokenize(cleanText)
#   fdist = FreqDist(arrayOfWords)
#   uniqueWords = []
#   for word in arrayOfWords:
#     if not word in uniqueWords:
#       uniqueWords.append(word)
#   for word in uniqueWords:
#     print(word, fdist[word])

# print(generateHaiku("the"))
# print("***************")
# print(generateHaiku("lksdjfhlsdhjfgsl;d"))
# print(identifyPartsOfSpeech("and"))

print(generateHaiku("miserable"))

# index of the parts of speech tags outputted by identifyingPartsOfSpeech() method
# http://www.scs.leeds.ac.uk/amalgam/tagsets/brown.html

# NOTES FROM MATT BAKER ABOUT REFACTORING
# def validThing(unigram):
# 	pos = str(identifyPartsOfSpeech(unigram.word2))
# 	countSyllables(unigram.word2) == 1 &&
# 	pos != 'IN' && pos != "DT"
# def formatPossibleWords(unigrams, reqSylCount):
# 	tempContainer = []
# 	container = []
# 	if reqSylCount == 1:
		# tempContainer = [word in unigrams if validThing(unigram)]
		# for each in unigrams:

<commit_msg>comment out non production print statement<commit_after>import nltk
from nltk.corpus import cmudict
import curses
from curses.ascii import isdigit
import re
from nltk.probability import FreqDist
from app import db
import models
from random import randrange

d = cmudict.dict()

def numSylsInWord(word):
	if word.lower() in d:
		return [len(list(y for y in x if y[-1].isdigit())) for x in d[word.lower()]][0]

def isHaiku(potentialHaiku):
	syllableCount = countSyllables(potentialHaiku)
	if syllableCount == 17:
		#how do we want to break this down?
		#what are the other "ifs" that this conditional needs to meet?
		result = True
	else:
		result = False
	return result

def countSyllables(potentialHaiku):
	stripPunctuation = re.sub(ur"[^\w\d'\s]+",' ',potentialHaiku)
	wordsInHaiku = stripPunctuation.split()
	syllableCount = 0
	for i in wordsInHaiku:
		syllableCount += numSylsInWord(i)
	return syllableCount

def inDatabase(firstWord):
	container = []
	from models import Unigram
	unigrams = Unigram.query.filter(Unigram.word1 == firstWord)
	for each in unigrams:
			for unigram in range(each.count):
					container.append(each.word2)
	if not container:
		return False
	if container:
		return True

def generateHaiku(firstWord):
	inDB = inDatabase(firstWord)
	if inDB:
		haiku = ""
		haiku += startGenerateLine(5, firstWord)
		haiku += "\n"
		haiku += startGenerateLine(7)
		haiku += "\n"
		haiku += startGenerateLine(5)
	if not inDB:
		firstWord = pickRandomWord(5)
		haiku = generateHaiku(firstWord)
	return haiku

def startGenerateLine(sylCount, startingWord= None):
	if not startingWord:
		startingWord = pickRandomWord(sylCount)
	remainingSylCount = sylCount - countSyllables(startingWord)
	line = buildLineList(remainingSylCount, [startingWord])
	return " ".join(line)

def buildLineList(sylCount, wordsFromBefore):
	if sylCount == 0:
		return wordsFromBefore
	lastWord = wordsFromBefore[-1]
	possibilities = grabPossibleWords(lastWord, sylCount)
	for possibleWord in possibilities:
		newWordsFromBefore = [word[:] for word in wordsFromBefore]
		newWordsFromBefore.append(possibleWord)
		newSyllableCount = sylCount - countSyllables(possibleWord)
		result = buildLineList(newSyllableCount, newWordsFromBefore)
		if result:
			return result
	return None

def pickRandomWord(reqSylCount):
	from models import Unigram
	lengthDB = Unigram.query.count()
	while True:
		randomNumPick = randrange(1, lengthDB)
		tryWord = Unigram.query.filter(Unigram.id == randomNumPick).first()
		if countSyllables(tryWord.word1) <= reqSylCount:
			word = tryWord.word1
			break
	return tryWord.word1

def formatPossibleWords(unigrams, reqSylCount):
	tempContainer = []
	container = []
	if reqSylCount == 1 or reqSylCount == 2:
		for each in unigrams:
			tempContainer.append(each)
		newTempContainer = removePartOfSpeech('IN', tempContainer)
		nextTempContainer = removePartOfSpeech('CC', newTempContainer)
		finalTempContainer = removePartOfSpeech('DT', nextTempContainer)
		for unigram in finalTempContainer:
			for index in range(unigram.count):
				if countSyllables(unigram.word2) <= reqSylCount:
					container.append(unigram.word2)
	else:
		for each in unigrams:
			for index in range(each.count):
				if countSyllables(each.word2) <= reqSylCount:
					container.append(each.word2)
	return container

def removePartOfSpeech(pos, tempContainer):
	for unigram in tempContainer:
		if identifyPartsOfSpeech(unigram.word2) == pos:
			tempContainer.remove(unigram)
	return tempContainer

def grabPossibleWords(baseWord, reqSylCount):
	from models import Unigram
	listOfUnigrams = Unigram.query.filter(Unigram.word1 ==baseWord)
	return formatPossibleWords(listOfUnigrams, reqSylCount)

def identifyPartsOfSpeech(word):
	cleanString = re.sub(ur"[^\w\d'\s]+",' ', word)
	pos = nltk.word_tokenize(cleanString)
	result = nltk.pos_tag(pos)
	return result[0][1]

# def findFrequency(largeBodyofText):
#   cleanText = re.sub(ur"[^\w\d'\s]+",' ', largeBodyofText)
#   arrayOfWords = nltk.word_tokenize(cleanText)
#   fdist = FreqDist(arrayOfWords)
#   uniqueWords = []
#   for word in arrayOfWords:
#     if not word in uniqueWords:
#       uniqueWords.append(word)
#   for word in uniqueWords:
#     print(word, fdist[word])

# print(generateHaiku("the"))
# print("***************")
# print(generateHaiku("lksdjfhlsdhjfgsl;d"))
# print(identifyPartsOfSpeech("and"))

# print(generateHaiku("miserable"))

# index of the parts of speech tags outputted by identifyingPartsOfSpeech() method
# http://www.scs.leeds.ac.uk/amalgam/tagsets/brown.html

# NOTES FROM MATT BAKER ABOUT REFACTORING
# def validThing(unigram):
# 	pos = str(identifyPartsOfSpeech(unigram.word2))
# 	countSyllables(unigram.word2) == 1 &&
# 	pos != 'IN' && pos != "DT"
# def formatPossibleWords(unigrams, reqSylCount):
# 	tempContainer = []
# 	container = []
# 	if reqSylCount == 1:
		# tempContainer = [word in unigrams if validThing(unigram)]
		# for each in unigrams:

<|endoftext|>


<commit_before>#! /usr/bin/env python2.7
import math
import subprocess
import os, glob, shutil
import errno
import sys
from argparse import ArgumentParser
import stat
import cantera as ct
import numpy as np
import multiprocessing

def check_dir(dir, force):
    old_files = [file for file in os.listdir(dir) if '.timing' in file and os.path.isfile(os.path.join(dir, file))]
    if len(old_files) and not force:
        raise Exception("Old data found in /{}/... stopping".format(dir))

def make_sure_path_exists(path):
    try:
        os.makedirs(path)
    except OSError as exception:
        if exception.errno != errno.EEXIST:
            raise

def get_executables(blacklist, inverse=None):
    exes = []
    if inverse is None:
        inverse = []
    executable = stat.S_IEXEC | stat.S_IXGRP | stat.S_IXOTH
    for filename in os.listdir('.'):
        if os.path.isfile(filename):
            st = os.stat(filename)
            mode = st.st_mode
            if mode & executable:
                if not any(b in filename for b in blacklist) \
                    and all(i in filename for i in inverse):
                    exes.append(filename)
    return exes

def get_powers(num_cond):
    powers = [1]
    while powers[-1] < num_cond:
        powers.append(powers[-1] * 2)
    if powers[-1] != num_cond:
        powers.append(num_cond)
    return powers

def get_diff_ics_cond(thedir, mechanism):
    gas = ct.Solution(mechanism)
    data = np.fromfile(os.path.join(thedir, 'data.bin'), dtype='float64')
    num_c = data.shape[0] / float(gas.n_species + 3)
    assert int(num_c) == num_c
    return int(num_c)

def run(thedir, run_me, force=False, pyjac='', repeats=5, num_cond=131072):
    jthread = str(multiprocessing.cpu_count())

    mechanism = os.path.join(thedir, glob.glob(os.path.join(thedir, '*.cti'))[0])
    with open(os.path.join(thedir, 'ics.txt'), 'r') as file:
        ic_str = file.readline().strip()

    home = os.getcwd()
    threads = [6, 12]
    same_powers = get_powers(num_cond)
    diff_powers = get_powers(get_diff_ics_cond(thedir, mechanism))

    #copy the datafile
    shutil.copy(os.path.join(thedir, 'data.bin'),
                os.path.join(home, 'ign_data.bin'))

    #generate mechanisms
    cache_opt = [True, False]
    use_smem = [True, False]
    same_ics = [True, False]
    for opt in cache_opt:
        mech_dir = 'cpu_{}'.format('co' if cache_opt else 'nco')
        mech_dir = os.path.join(thedir, mech_dir) + os.path.sep
        make_sure_path_exists(mech_dir)
        args = ['python2.7', os.path.join(pyjac, 'pyJac.py'), '-l', 'c', '-i', mechanism]
        if not opt:
            args.append('-nco')
        args.extend(['-ic', ic_str])
        args.extend(['-b', mech_dir])
        subprocess.check_call(args)

        for smem in use_smem:
            gpu_mech_dir = 'gpu_{}_{}'.format('co' if cache_opt else 'nco', 'smem' if smem else 'nosmem')
            gpu_mech_dir = os.path.join(home, gpu_mech_dir)
            make_sure_path_exists(gpu_mech_dir)
            if opt:
                #copy the pickle from the cpu folder
                shutil.copy(os.path.join(mech_dir, 'optimized.pickle'),
                    os.path.join(gpu_mech_dir, 'optimized.pickle'))
            args = ['python2.7', os.path.join(pyjac, 'pyJac.py'), '-l', 'cuda', '-i', mechanism]
            if not opt:
                args.append('-nco')
            if not smem:
                args.append('-nosmem')
            args.extend(['-ic', ic_str])
            args.extend(['-b', mech_dir])
            subprocess.check_call(args)

    #now build and run
    for same in same_ics:
        thepow = same_powers if same else diff_powers
        for opt in cache_opt:
            mech_dir = 'cpu_{}'.format('co' if cache_opt else 'nco')
            mech_dir = os.path.join(thedir, mech_dir) + os.path.sep
            args = ['scons', 'cpu', '-j', jthread, 'DEBUG=False', 'FAST_MATH=FALSE',
                 'LOG_OUTPUT=FALSE','SHUFFLE=FALSE',
                 'PRINT=FALSE', 'mechanism_dir={}'.format(mech_dir)]
            args.append('SAME_IC={}'.format(same))
            #rebuild for performance
            subprocess.check_call(args)
            #run with repeats
            for exe in run_me:
                for thread in threads:
                    for cond in thepow:
                        with open(exe + '_{}_{}_{}.txt'.format(cond, thread,
                            'co' if cache_opt else 'nco'), 'a') as file:
                            for repeat in range(repeats):
                                subprocess.check_call([exe, str(thread), str(cond)], stdout=file)

            for smem in use_smem:
                gpu_mech_dir = 'gpu_{}_{}'.format('co' if cache_opt else 'nco', 'smem' if smem else 'nosmem')
                gpu_mech_dir = os.path.join(home, gpu_mech_dir)
                args = ['scons', 'gpu', '-j', jthread, 'DEBUG=False', 'FAST_MATH=FALSE',
                 'LOG_OUTPUT=FALSE','SHUFFLE=FALSE',
                 'PRINT=FALSE', 'mechanism_dir={}'.format(gpu_mech_dir)]
                args.append('SAME_IC={}'.format(same))
                subprocess.check_call(args)
                #run with repeats
                for exe in run_me:
                    for cond in thepow:
                        with open(exe + '{}_{}_{}.txt'.format(cond,
                            'co' if cache_opt else 'nco', 'smem' if smem else 'nosmem'), 'a') as file:
                            for repeat in range(repeats):
                                subprocess.check_call([exe, str(cond)], stdout=file)

if __name__ == '__main__':
    parser = ArgumentParser(description='Runs timing runs for the various integrators')
    parser.add_argument('-f', '--force',
                        required=False,
                        default=False,
                        action='store_true',
                        help='Force reuse of past data files')
    parser.add_argument('-b', '--base_dir',
                        required=False,
                        default='performance',
                        help='The base directory containing a folder per mechanism')
    parser.add_argument('-p', '--pyjac_dir',
                        required=False,
                        default='~/pyJac/',
                        help='The base directory of pyJac')
    parser.add_argument('-s', '--solver_blacklist',
                        required=False,
                        default='',
                        help='The solvers to not run')
    parser.add_argument('-n', '--num_cond',
                        type=int,
                        required=False,
                        default=131072,
                        help='The number of conditions to run for the same ics')
    parser.add_argument('-r', '--repeats',
                        required=False,
                        type=int,
                        default=5,
                        help='The number of timing repeats to run')
    args = parser.parse_args()

    home = os.getcwd()
    a_dir = os.path.join(os.getcwd(), args.base_dir)
    dir_list = sorted([os.path.join(a_dir, name) for name in os.listdir(a_dir)
            if os.path.isdir(os.path.join(a_dir, name))])

    run_me = get_executables(args.solver_blacklist.split(','))
    for d in dir_list:
        run(d, run_me, force=args.force,
            pyjac=os.path.expanduser(args.pyjac_dir),
            num_cond=args.num_cond,
            repeats=args.repeats)
        os.chdir(home)<commit_msg>fix to runner for new iteration scheme<commit_after>#! /usr/bin/env python2.7
import math
import subprocess
import os, glob, shutil
import errno
import sys
from argparse import ArgumentParser
import stat
import cantera as ct
import numpy as np
import multiprocessing

def check_dir(dir, force):
    old_files = [file for file in os.listdir(dir) if '.timing' in file and os.path.isfile(os.path.join(dir, file))]
    if len(old_files) and not force:
        raise Exception("Old data found in /{}/... stopping".format(dir))

def make_sure_path_exists(path):
    try:
        os.makedirs(path)
    except OSError as exception:
        if exception.errno != errno.EEXIST:
            raise

def get_executables(blacklist, inverse=None):
    exes = []
    if inverse is None:
        inverse = []
    executable = stat.S_IEXEC | stat.S_IXGRP | stat.S_IXOTH
    for filename in os.listdir(os.getcwd()):
        if os.path.isfile(filename):
            st = os.stat(filename)
            mode = st.st_mode
            if mode & executable:
                if not any(b in filename for b in blacklist) \
                    and all(i in filename for i in inverse):
                    exes.append(filename)
    return exes

def get_powers(num_cond):
    powers = [1]
    while powers[-1] < num_cond:
        powers.append(powers[-1] * 2)
    if powers[-1] != num_cond:
        powers.append(num_cond)
    return powers

def get_diff_ics_cond(thedir, mechanism):
    gas = ct.Solution(mechanism)
    data = np.fromfile(os.path.join(thedir, 'data.bin'), dtype='float64')
    num_c = data.shape[0] / float(gas.n_species + 3)
    assert int(num_c) == num_c
    return int(num_c)

def run(thedir, run_me, force=False, pyjac='', repeats=5, num_cond=131072):
    jthread = str(multiprocessing.cpu_count())

    mechanism = os.path.join(thedir, glob.glob(os.path.join(thedir, '*.cti'))[0])
    with open(os.path.join(thedir, 'ics.txt'), 'r') as file:
        ic_str = file.readline().strip()

    home = os.getcwd()
    threads = [6, 12]
    same_powers = get_powers(num_cond)
    diff_powers = get_powers(get_diff_ics_cond(thedir, mechanism))

    #copy the datafile
    shutil.copy(os.path.join(thedir, 'data.bin'),
                os.path.join(home, 'ign_data.bin'))

    #generate mechanisms
    cache_opt = [True, False]
    use_smem = [True, False]
    same_ics = [True, False]
    for opt in cache_opt:
        mech_dir = 'cpu_{}'.format('co' if cache_opt else 'nco')
        mech_dir = os.path.join(thedir, mech_dir) + os.path.sep
        make_sure_path_exists(mech_dir)
        args = ['python2.7', os.path.join(pyjac, 'pyJac.py'), '-l', 'c', '-i', mechanism]
        if not opt:
            args.append('-nco')
        args.extend(['-ic', ic_str])
        args.extend(['-b', mech_dir])
        subprocess.check_call(args)

        for smem in use_smem:
            gpu_mech_dir = 'gpu_{}_{}'.format('co' if cache_opt else 'nco', 'smem' if smem else 'nosmem')
            gpu_mech_dir = os.path.join(thedir, gpu_mech_dir)
            make_sure_path_exists(gpu_mech_dir)
            if opt:
                #copy the pickle from the cpu folder
                shutil.copy(os.path.join(mech_dir, 'optimized.pickle'),
                    os.path.join(gpu_mech_dir, 'optimized.pickle'))
            args = ['python2.7', os.path.join(pyjac, 'pyJac.py'), '-l', 'cuda', '-i', mechanism]
            if not opt:
                args.append('-nco')
            if not smem:
                args.append('-nosmem')
            args.extend(['-ic', ic_str])
            args.extend(['-b', mech_dir])
            subprocess.check_call(args)

    #now build and run
    for same in same_ics:
        thepow = same_powers if same else diff_powers
        for opt in cache_opt:
            mech_dir = 'cpu_{}'.format('co' if cache_opt else 'nco')
            mech_dir = os.path.join(thedir, mech_dir) + os.path.sep
            args = ['scons', 'cpu', '-j', jthread, 'DEBUG=False', 'FAST_MATH=FALSE',
                 'LOG_OUTPUT=FALSE','SHUFFLE=FALSE',
                 'PRINT=FALSE', 'mechanism_dir={}'.format(mech_dir)]
            args.append('SAME_IC={}'.format(same))
            #rebuild for performance
            subprocess.check_call(args)
            #run with repeats
            for exe in run_me:
                for thread in threads:
                    for cond in thepow:
                        with open(exe + '_{}_{}_{}.txt'.format(cond, thread,
                            'co' if cache_opt else 'nco'), 'a') as file:
                            for repeat in range(repeats):
                                subprocess.check_call([exe, str(thread), str(cond)], stdout=file)

            for smem in use_smem:
                gpu_mech_dir = 'gpu_{}_{}'.format('co' if cache_opt else 'nco', 'smem' if smem else 'nosmem')
                gpu_mech_dir = os.path.join(thedir, gpu_mech_dir)
                args = ['scons', 'gpu', '-j', jthread, 'DEBUG=False', 'FAST_MATH=FALSE',
                 'LOG_OUTPUT=FALSE','SHUFFLE=FALSE',
                 'PRINT=FALSE', 'mechanism_dir={}'.format(gpu_mech_dir)]
                args.append('SAME_IC={}'.format(same))
                subprocess.check_call(args)
                #run with repeats
                for exe in run_me:
                    for cond in thepow:
                        with open(exe + '{}_{}_{}.txt'.format(cond,
                            'co' if cache_opt else 'nco', 'smem' if smem else 'nosmem'), 'a') as file:
                            for repeat in range(repeats):
                                subprocess.check_call([exe, str(cond)], stdout=file)

if __name__ == '__main__':
    parser = ArgumentParser(description='Runs timing runs for the various integrators')
    parser.add_argument('-f', '--force',
                        required=False,
                        default=False,
                        action='store_true',
                        help='Force reuse of past data files')
    parser.add_argument('-b', '--base_dir',
                        required=False,
                        default='performance',
                        help='The base directory containing a folder per mechanism')
    parser.add_argument('-p', '--pyjac_dir',
                        required=False,
                        default='~/pyJac/',
                        help='The base directory of pyJac')
    parser.add_argument('-s', '--solver_blacklist',
                        required=False,
                        default='',
                        help='The solvers to not run')
    parser.add_argument('-n', '--num_cond',
                        type=int,
                        required=False,
                        default=131072,
                        help='The number of conditions to run for the same ics')
    parser.add_argument('-r', '--repeats',
                        required=False,
                        type=int,
                        default=5,
                        help='The number of timing repeats to run')
    args = parser.parse_args()

    home = os.getcwd()
    a_dir = os.path.join(os.getcwd(), args.base_dir)
    dir_list = sorted([os.path.join(a_dir, name) for name in os.listdir(a_dir)
            if os.path.isdir(os.path.join(a_dir, name))])

    run_me = get_executables(args.solver_blacklist.split(','))
    for d in dir_list:
        run(d, run_me, force=args.force,
            pyjac=os.path.expanduser(args.pyjac_dir),
            num_cond=args.num_cond,
            repeats=args.repeats)<|endoftext|>


<commit_before>#!/usr/bin/env python
"""runner [-v|-q] [-c config] taskdir"""
import os
import time
import subprocess
import re
import sys
from ConfigParser import RawConfigParser

import logging
log = logging.getLogger(__name__)

def run_task(t, env, max_time):
    start = time.time()
    proc = subprocess.Popen(t, stdin=open(os.devnull, 'r'), env=env)
    while True:
        if proc.poll() is not None:
            break

        if time.time() - start > max_time:
            # Try killing it
            log.warn("exceeded max_time; killing")
            proc.terminate()
            return "RETRY"
        else:
            time.sleep(1)

    rv = proc.wait()
    if rv == 0:
        return "OK"
    elif rv == 2:
        return "HALT"
    else:
        return "RETRY"

class Config(object):
    sleep_time = 1
    max_tries = 5
    max_time = 600
    halt_task = "halt.sh"
    filename = None
    options = None

    def load_config(self, filename):
        self.filename = filename
        self.options = RawConfigParser()
        # The default optionxform converts option names to lower case. We want
        # to preserve case, so change the transform function to just return the
        # str value
        self.options.optionxform = str
        if not self.options.read([filename]):
            log.warn("Couldn't load %s", filename)
            self.options = None
            return
        if self.options.has_option('runner', 'include_dir'):
            # reload the object including files in config.d
            config_dir = self.options.get('runner', 'include_dir')
            if not self.options.read([filename] + list_directory(config_dir)):
                log.warn("Couldn't load %s", config_dir)
                return

        if self.options.has_option('runner', 'sleep_time'):
            self.sleep_time = self.options.getint('runner', 'sleep_time')
        if self.options.has_option('runner', 'max_tries'):
            self.max_tries = self.options.getint('runner', 'max_tries')
        if self.options.has_option('runner', 'max_time'):
            self.max_time = self.options.getint('runner', 'max_time')
        if self.options.has_option('runner', 'halt_task'):
            self.halt_task = self.options.get('runner', 'halt_task')

    def get(self, section, option):
        if self.options and self.options.has_option(section, option):
            return self.options.get(section, option)
        return None

    def get_env(self):
        retval = {}
        if self.options and self.options.has_section('env'):
            for option, value in self.options.items('env'):
                retval[str(option)] = str(value)
        if self.filename:
            retval['RUNNER_CONFIG_CMD'] = '{python} {runner} -c {configfile}'.format(
                python=sys.executable,
                runner=os.path.abspath(sys.argv[0]),
                configfile=os.path.abspath(self.filename),
            )
        return retval

def maybe_int(x):
    """Returns int(x), or x if x can't be converted to an int

    >>> maybe_int(1)
    1

    >>> maybe_int('1')
    1

    >>> maybe_int('notanint')
    'notanint'
    """

    try:
        return int(x)
    except ValueError:
        return x

def naturalsort_key(x):
    """
    Splits x into numbers/not-numbers so it can be sorted

    >>> naturalsort_key('1-foo.bar')
    (1, '-foo.bar')

    >>> naturalsort_key('11-foo.bar')
    (11, '-foo.bar')
    """
    return tuple(maybe_int(y) for y in filter(None, re.split("(\d+)", x)))

def naturally_sorted(l):
    """
    Returns list l sorted naturally

    >>> naturally_sorted(reversed(['0-first', '1-second', '2-third', '11-fourth']))
    ['0-first', '1-second', '2-third', '11-fourth']
    """
    return sorted(l, key=naturalsort_key)

def list_directory(dirname):
    # List the files in the directory, and sort them
    files = naturally_sorted(os.listdir(dirname))
    # Filter out files with leading .
    return [f for f in files if f[0] != '.']

def process_taskdir(config, dirname):
    tasks = list_directory(dirname)
    # Filter out the halting task
    if config.halt_task in tasks:
        tasks.remove(config.halt_task)

    log.debug("tasks: %s", tasks)

    env = os.environ.copy()
    new_env = config.get_env()
    log.debug("Updating env with %s", new_env)
    env.update(new_env)

    max_time = config.max_time

    for try_num in range(1, config.max_tries + 1):
        for t in tasks:
            log.debug("%s: starting (max time %is)", t, max_time)
            r = run_task(os.path.join(dirname, t), env, max_time=max_time)
            log.debug("%s: %s", t, r)
            if r == "OK":
                continue
            elif r == "RETRY":
                # No point in sleeping if we're on our last try
                if try_num == config.max_tries:
                    log.warn("maximum attempts reached")
                    # TODO: halt here too?
                    return False
                # Sleep and try again
                log.debug("sleeping for %i", config.sleep_time)
                time.sleep(config.sleep_time)
                break
            elif r == "HALT":
                # stop/halt/reboot?
                log.info("halting")
                run_task(os.path.join(dirname, config.halt_task), env, max_time=max_time)
                return False
        else:
            log.debug("all tasks completed!")
            return True

def make_argument_parser():
    import argparse
    parser = argparse.ArgumentParser(__doc__)
    parser.set_defaults(
        loglevel=logging.INFO,
    )
    parser.add_argument("-q", "--quiet", dest="loglevel", action="store_const", const=logging.WARN, help="quiet")
    parser.add_argument("-v", "--verbose", dest="loglevel", action="store_const", const=logging.DEBUG, help="verbose")
    parser.add_argument("-c", "--config", dest="config_file")
    parser.add_argument("-g", "--get", dest="get", help="get configuration value")
    parser.add_argument("-n", "--times", dest="times", type=int, help="run this many times (default is forever)")
    parser.add_argument("taskdir", help="task directory", nargs="?")

    return parser

def runner(config, taskdir, times):
    """Runs tasks in the taskdir up to `times` number of times

    times can be None to run forever
    """
    t = 0
    while True:
        t += 1
        log.info("iteration %i", t)
        if times and t > times:
            break
        if not process_taskdir(config, taskdir):
            exit(1)

def main():
    parser = make_argument_parser()
    args = parser.parse_args()
    logging.basicConfig(format="%(asctime)s - %(levelname)s - %(message)s", level=args.loglevel)

    config = Config()
    if args.config_file:
        config.load_config(args.config_file)

    if args.get:
        log.debug("getting %s", args.get)
        section, option = args.get.split(".", 1)
        v = config.get(section, option)
        if v is not None:
            print v
        exit(0)
    elif not args.taskdir:
        parser.error("taskdir required")

    if not os.path.exists(args.taskdir):
        log.error("%s doesn't exist", args.taskdir)
        exit(1)

    runner(config, args.taskdir, args.times)

if __name__ == '__main__':
    main()
<commit_msg>Properly include files in config_dir<commit_after>#!/usr/bin/env python
"""runner [-v|-q] [-c config] taskdir"""
import os
import time
import subprocess
import re
import sys
from ConfigParser import RawConfigParser

import logging
log = logging.getLogger(__name__)

def run_task(t, env, max_time):
    start = time.time()
    proc = subprocess.Popen(t, stdin=open(os.devnull, 'r'), env=env)
    while True:
        if proc.poll() is not None:
            break

        if time.time() - start > max_time:
            # Try killing it
            log.warn("exceeded max_time; killing")
            proc.terminate()
            return "RETRY"
        else:
            time.sleep(1)

    rv = proc.wait()
    if rv == 0:
        return "OK"
    elif rv == 2:
        return "HALT"
    else:
        return "RETRY"

class Config(object):
    sleep_time = 1
    max_tries = 5
    max_time = 600
    halt_task = "halt.sh"
    filename = None
    options = None

    def load_config(self, filename):
        self.filename = filename
        self.options = RawConfigParser()
        # The default optionxform converts option names to lower case. We want
        # to preserve case, so change the transform function to just return the
        # str value
        self.options.optionxform = str
        if not self.options.read([filename]):
            log.warn("Couldn't load %s", filename)
            self.options = None
            return
        if self.options.has_option('runner', 'include_dir'):
            # reload the object including files in config.d
            config_dir = self.options.get('runner', 'include_dir')
            configs = [os.path.join(config_dir, c) for c in list_directory(config_dir)]
            if not self.options.read([filename] + configs):
                log.warn("Couldn't load %s", config_dir)
                return

        if self.options.has_option('runner', 'sleep_time'):
            self.sleep_time = self.options.getint('runner', 'sleep_time')
        if self.options.has_option('runner', 'max_tries'):
            self.max_tries = self.options.getint('runner', 'max_tries')
        if self.options.has_option('runner', 'max_time'):
            self.max_time = self.options.getint('runner', 'max_time')
        if self.options.has_option('runner', 'halt_task'):
            self.halt_task = self.options.get('runner', 'halt_task')

    def get(self, section, option):
        if self.options and self.options.has_option(section, option):
            return self.options.get(section, option)
        return None

    def get_env(self):
        retval = {}
        if self.options and self.options.has_section('env'):
            for option, value in self.options.items('env'):
                retval[str(option)] = str(value)
        if self.filename:
            retval['RUNNER_CONFIG_CMD'] = '{python} {runner} -c {configfile}'.format(
                python=sys.executable,
                runner=os.path.abspath(sys.argv[0]),
                configfile=os.path.abspath(self.filename),
            )
        return retval

def maybe_int(x):
    """Returns int(x), or x if x can't be converted to an int

    >>> maybe_int(1)
    1

    >>> maybe_int('1')
    1

    >>> maybe_int('notanint')
    'notanint'
    """

    try:
        return int(x)
    except ValueError:
        return x

def naturalsort_key(x):
    """
    Splits x into numbers/not-numbers so it can be sorted

    >>> naturalsort_key('1-foo.bar')
    (1, '-foo.bar')

    >>> naturalsort_key('11-foo.bar')
    (11, '-foo.bar')
    """
    return tuple(maybe_int(y) for y in filter(None, re.split("(\d+)", x)))

def naturally_sorted(l):
    """
    Returns list l sorted naturally

    >>> naturally_sorted(reversed(['0-first', '1-second', '2-third', '11-fourth']))
    ['0-first', '1-second', '2-third', '11-fourth']
    """
    return sorted(l, key=naturalsort_key)

def list_directory(dirname):
    # List the files in the directory, and sort them
    files = naturally_sorted(os.listdir(dirname))
    # Filter out files with leading .
    return [f for f in files if f[0] != '.']

def process_taskdir(config, dirname):
    tasks = list_directory(dirname)
    # Filter out the halting task
    if config.halt_task in tasks:
        tasks.remove(config.halt_task)

    log.debug("tasks: %s", tasks)

    env = os.environ.copy()
    new_env = config.get_env()
    log.debug("Updating env with %s", new_env)
    env.update(new_env)

    max_time = config.max_time

    for try_num in range(1, config.max_tries + 1):
        for t in tasks:
            log.debug("%s: starting (max time %is)", t, max_time)
            r = run_task(os.path.join(dirname, t), env, max_time=max_time)
            log.debug("%s: %s", t, r)
            if r == "OK":
                continue
            elif r == "RETRY":
                # No point in sleeping if we're on our last try
                if try_num == config.max_tries:
                    log.warn("maximum attempts reached")
                    # TODO: halt here too?
                    return False
                # Sleep and try again
                log.debug("sleeping for %i", config.sleep_time)
                time.sleep(config.sleep_time)
                break
            elif r == "HALT":
                # stop/halt/reboot?
                log.info("halting")
                run_task(os.path.join(dirname, config.halt_task), env, max_time=max_time)
                return False
        else:
            log.debug("all tasks completed!")
            return True

def make_argument_parser():
    import argparse
    parser = argparse.ArgumentParser(__doc__)
    parser.set_defaults(
        loglevel=logging.INFO,
    )
    parser.add_argument("-q", "--quiet", dest="loglevel", action="store_const", const=logging.WARN, help="quiet")
    parser.add_argument("-v", "--verbose", dest="loglevel", action="store_const", const=logging.DEBUG, help="verbose")
    parser.add_argument("-c", "--config", dest="config_file")
    parser.add_argument("-g", "--get", dest="get", help="get configuration value")
    parser.add_argument("-n", "--times", dest="times", type=int, help="run this many times (default is forever)")
    parser.add_argument("taskdir", help="task directory", nargs="?")

    return parser

def runner(config, taskdir, times):
    """Runs tasks in the taskdir up to `times` number of times

    times can be None to run forever
    """
    t = 0
    while True:
        t += 1
        log.info("iteration %i", t)
        if times and t > times:
            break
        if not process_taskdir(config, taskdir):
            exit(1)

def main():
    parser = make_argument_parser()
    args = parser.parse_args()
    logging.basicConfig(format="%(asctime)s - %(levelname)s - %(message)s", level=args.loglevel)

    config = Config()
    if args.config_file:
        config.load_config(args.config_file)

    if args.get:
        log.debug("getting %s", args.get)
        section, option = args.get.split(".", 1)
        v = config.get(section, option)
        if v is not None:
            print v
        exit(0)
    elif not args.taskdir:
        parser.error("taskdir required")

    if not os.path.exists(args.taskdir):
        log.error("%s doesn't exist", args.taskdir)
        exit(1)

    runner(config, args.taskdir, args.times)

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from __future__ import print_function

import os
import glob
import logging
import re
import subprocess
import tempfile
from datetime import datetime as dt
from string import Template

class TaskFailedException(Exception):
    pass

class TempDirIsFileException(Exception):
    pass

class Task(object):

    __dirname = ".faz"
    __variable_pattern = re.compile(r"\$\[([a-zA-Z0-9_]+)\]")

    def __init__(self, inputs, outputs, code, options, environment):
        self.inputs = inputs
        self.outputs = outputs
        self.original_outputs = outputs
        self.code = code
        self.options = options
        self.environment = environment
        self.order = 0
        self.force = False
        self.interpreter = None
        self.f = None
        self.check_options()

    def check_options(self):
        interpreter = "bash"
        for option in self.options:
            if "python" in option.lower():
                interpreter = "python"
            elif "ruby" in option.lower():
                interpreter = "ruby"
            elif "force" in option.lower():
                self.force = True
        self.interpreter = interpreter

    def check_inputs(self):
        """ Check for the existence of input files """
        self.inputs = self.expand_filenames(self.inputs)
        result = False
        if len(self.inputs) == 0 or self.files_exist(self.inputs):
            result = True
        else:
            print("Not executing task. Input file(s) do not exist.")
        return result

    def check_outputs(self):
        """ Check for the existence of output files """
        self.outputs = self.expand_filenames(self.outputs)
        result = False
        if self.files_exist(self.outputs):
            if self.dependencies_are_newer(self.outputs, self.inputs):
                result = True
                print("Dependencies are newer than outputs.")
                print("Running task.")
            elif self.force:
                print("Dependencies are older than inputs, but 'force' option present.")
                print("Running task.")
                result = True
            else:
                print("Dependencies are older than inputs.")
        else:
            print("No ouput file(s).")
            print("Running task.")
            result = True
        return result

    def expand_variables(self):
        """
        Expand variables in the task code.
        Only variables who use the $[<variable name>] format are expanded.
        Variables using the $<variable name> and ${<variable name>} formats
        are expanded by the shell (in the cases where bash is the interpreter.
        """
        for n, input_file in enumerate(self.inputs):
            self.environment["INPUT{}".format(n +1)] = input_file
        for n, output_file in enumerate(self.outputs):
            self.environment["OUTPUT{}".format(n +1)] = output_file
        for n, line in enumerate(self.code):
            match = self.__variable_pattern.findall(line)
            if len(match) > 0:
                for item in match:
                    value = self.environment.get(item)
                    if value is not None:
                        self.code[n] = self.code[n].replace("$[" + item + "]", value)

    def expand_filenames(self, filenames):
        """
        Expand a list of filenames using environment variables,
        followed by expansion of shell-style wildcards.
        """
        results = []
        for filename in filenames:
            result = filename
            if "$" in filename:
                template = Template(filename)
                result = template.substitute(**self.environment)
                logging.debug(
                    "Expanding {} to {}.".format(filename, result))
            if any([pattern in result for pattern in "*[]?"]):
                expanded = glob.glob(result)
                if len(expanded) > 0:
                    result = expanded
                else:
                    result = "NONEXISTENT"
            if isinstance(result, list):
                results.extend(result)
            else:
                results.append(result)
        return sorted(list(set(results)))

    def files_exist(self, filenames):
        """ Check if all files in a given list exist. """
        return all([os.path.exists(filename) and os.path.isfile(filename)
                    for filename in filenames])

    def dependencies_are_newer(self, files, dependencies):
        """
        For two lists of files, check if any file in the
        second list is newer than any file of the first.
        """
        dependency_mtimes = [
            os.path.getmtime(filename) for filename in dependencies]
        file_mtimes = [os.path.getmtime(filename) for filename in files]
        result = False
        for file_mtime in file_mtimes:
            for dependency_mtime in dependency_mtimes:
                if dependency_mtime > file_mtime:
                    result = True
        return result

    def __call__(self):
        """ Invoque an interpreter to execute the task's code. """
        print("\n ********** Task {0}: {1}\n".format(self.order, self))
        if self.check_inputs() and self.check_outputs():
            self.expand_variables()
            self.mktemp_file()
            #os.write(self.f, "\n".join(self.code) + "\n")
            self.f.write("\n".join(self.code) + "\n")
            self.f.close()
            logging.debug("Environment before task:\n{}".format(self.environment))
            print("Task inputs: {}".format(self.inputs))
            print("Task outputs: {}".format(self.outputs))
            start = dt.now()
            try:
                out = subprocess.check_output([self.interpreter, self.f.name],
                                              stderr=subprocess.STDOUT,
                                              env=self.environment)
            except subprocess.CalledProcessError as e:
                print("Task {} failed with return code {}".format(self, e.returncode))
                print("Output from task:")
                print(e.output)
                raise TaskFailedException("Task {} failed.".format(self))
            end = dt.now()
            logging.debug("Environment after task:\n{}".format(self.environment))
            print("***** execution time {}".format(str(end - start)))
            print("***** Output:\n{}".format(out))
            #os.unlink(self.f.name)
            self.outputs = self.expand_filenames(self.original_outputs)
            if not(self.files_exist(self.outputs)):
                print("Output files:")
                for filename in self.outputs:
                    print("{}: {}".format(filename, os.path.exists(filename)))
                raise TaskFailedException("Output files not created for Task {}\n".format(self))

    def mktemp_file(self):
        """ Create a temporary file in the '.faz' directory for
            the code to feed to the interpreter. """
        if not(os.path.exists(self.__dirname)):
            logging.debug("Creating directory {}".format(self.__dirname))
            os.mkdir(self.__dirname)
        elif not(os.path.isdir(self.__dirname)):
            raise TempDirIsFileException(
                "There is a file called %s in this directory!!!" %
                self.__dirname)
        #self.fdesc, self.fname = tempfile.mkstemp(dir=self.__dirname, text=True)
        self.f = tempfile.NamedTemporaryFile(dir=self.__dirname, delete=False, mode="wt")
        logging.debug("Creating file {}".format(self.f.name))

    def __repr__(self):
        return "%s <- %s :%s" % (
            ", ".join(self.outputs),
            ", ".join(self.inputs),
            ", ".join(self.options))
<commit_msg>Added INPUTS and OUTPUTS special variables to task environment.<commit_after># -*- coding: utf-8 -*-
from __future__ import print_function

import os
import glob
import logging
import re
import subprocess
import tempfile
from datetime import datetime as dt
from string import Template

class TaskFailedException(Exception):
    pass

class TempDirIsFileException(Exception):
    pass

class Task(object):

    __dirname = ".faz"
    __variable_pattern = re.compile(r"\$\[([a-zA-Z0-9_]+)\]")

    def __init__(self, inputs, outputs, code, options, environment):
        self.inputs = inputs
        self.outputs = outputs
        self.original_outputs = outputs
        self.code = code
        self.options = options
        self.environment = environment
        self.order = 0
        self.force = False
        self.interpreter = None
        self.f = None
        self.check_options()

    def check_options(self):
        interpreter = "bash"
        for option in self.options:
            if "python" in option.lower():
                interpreter = "python"
            elif "ruby" in option.lower():
                interpreter = "ruby"
            elif "force" in option.lower():
                self.force = True
        self.interpreter = interpreter

    def check_inputs(self):
        """ Check for the existence of input files """
        self.inputs = self.expand_filenames(self.inputs)
        result = False
        if len(self.inputs) == 0 or self.files_exist(self.inputs):
            result = True
        else:
            print("Not executing task. Input file(s) do not exist.")
        return result

    def check_outputs(self):
        """ Check for the existence of output files """
        self.outputs = self.expand_filenames(self.outputs)
        result = False
        if self.files_exist(self.outputs):
            if self.dependencies_are_newer(self.outputs, self.inputs):
                result = True
                print("Dependencies are newer than outputs.")
                print("Running task.")
            elif self.force:
                print("Dependencies are older than inputs, but 'force' option present.")
                print("Running task.")
                result = True
            else:
                print("Dependencies are older than inputs.")
        else:
            print("No ouput file(s).")
            print("Running task.")
            result = True
        return result

    def expand_variables(self):
        """
        Expand variables in the task code.
        Only variables who use the $[<variable name>] format are expanded.
        Variables using the $<variable name> and ${<variable name>} formats
        are expanded by the shell (in the cases where bash is the interpreter.
        """
        self.environment["INPUTS"] = self.inputs
        self.environment["OUTPUTS"] = self.outputs
        for n, input_file in enumerate(self.inputs):
            self.environment["INPUT{}".format(n +1)] = input_file
        for n, output_file in enumerate(self.outputs):
            self.environment["OUTPUT{}".format(n +1)] = output_file
        for n, line in enumerate(self.code):
            match = self.__variable_pattern.findall(line)
            if len(match) > 0:
                for item in match:
                    value = self.environment.get(item)
                    if value is not None:
                        self.code[n] = self.code[n].replace("$[" + item + "]", value)

    def expand_filenames(self, filenames):
        """
        Expand a list of filenames using environment variables,
        followed by expansion of shell-style wildcards.
        """
        results = []
        for filename in filenames:
            result = filename
            if "$" in filename:
                template = Template(filename)
                result = template.substitute(**self.environment)
                logging.debug(
                    "Expanding {} to {}.".format(filename, result))
            if any([pattern in result for pattern in "*[]?"]):
                expanded = glob.glob(result)
                if len(expanded) > 0:
                    result = expanded
                else:
                    result = "NONEXISTENT"
            if isinstance(result, list):
                results.extend(result)
            else:
                results.append(result)
        return sorted(list(set(results)))

    def files_exist(self, filenames):
        """ Check if all files in a given list exist. """
        return all([os.path.exists(filename) and os.path.isfile(filename)
                    for filename in filenames])

    def dependencies_are_newer(self, files, dependencies):
        """
        For two lists of files, check if any file in the
        second list is newer than any file of the first.
        """
        dependency_mtimes = [
            os.path.getmtime(filename) for filename in dependencies]
        file_mtimes = [os.path.getmtime(filename) for filename in files]
        result = False
        for file_mtime in file_mtimes:
            for dependency_mtime in dependency_mtimes:
                if dependency_mtime > file_mtime:
                    result = True
        return result

    def __call__(self):
        """ Invoque an interpreter to execute the task's code. """
        print("\n ********** Task {0}: {1}\n".format(self.order, self))
        if self.check_inputs() and self.check_outputs():
            self.expand_variables()
            self.mktemp_file()
            #os.write(self.f, "\n".join(self.code) + "\n")
            self.f.write("\n".join(self.code) + "\n")
            self.f.close()
            logging.debug("Environment before task:\n{}".format(self.environment))
            print("Task inputs: {}".format(self.inputs))
            print("Task outputs: {}".format(self.outputs))
            start = dt.now()
            try:
                out = subprocess.check_output([self.interpreter, self.f.name],
                                              stderr=subprocess.STDOUT,
                                              env=self.environment)
            except subprocess.CalledProcessError as e:
                print("Task {} failed with return code {}".format(self, e.returncode))
                print("Output from task:")
                print(e.output)
                raise TaskFailedException("Task {} failed.".format(self))
            end = dt.now()
            logging.debug("Environment after task:\n{}".format(self.environment))
            print("***** execution time {}".format(str(end - start)))
            print("***** Output:\n{}".format(out))
            #os.unlink(self.f.name)
            self.outputs = self.expand_filenames(self.original_outputs)
            if not(self.files_exist(self.outputs)):
                print("Output files:")
                for filename in self.outputs:
                    print("{}: {}".format(filename, os.path.exists(filename)))
                raise TaskFailedException("Output files not created for Task {}\n".format(self))

    def mktemp_file(self):
        """ Create a temporary file in the '.faz' directory for
            the code to feed to the interpreter. """
        if not(os.path.exists(self.__dirname)):
            logging.debug("Creating directory {}".format(self.__dirname))
            os.mkdir(self.__dirname)
        elif not(os.path.isdir(self.__dirname)):
            raise TempDirIsFileException(
                "There is a file called %s in this directory!!!" %
                self.__dirname)
        #self.fdesc, self.fname = tempfile.mkstemp(dir=self.__dirname, text=True)
        self.f = tempfile.NamedTemporaryFile(dir=self.__dirname, delete=False, mode="wt")
        logging.debug("Creating file {}".format(self.f.name))

    def __repr__(self):
        return "%s <- %s :%s" % (
            ", ".join(self.outputs),
            ", ".join(self.inputs),
            ", ".join(self.options))
<|endoftext|>