<commit_before>
import copy
import os
import re
import types

from quickrelease.config import ConfigSpecError, ConfigSpec

class Deliverable(object):
   DELIVERABLE_SECTION_PREFIX = 'deliverable'
   DELIVERABLE_CONFIG_PREFIX = (DELIVERABLE_SECTION_PREFIX +
    ConfigSpec.CONFIG_SECTION_DELIMETER)

   ERROR_STR_NEED_NAME_OR_REGEX = ("Deliverable class '%s' must define a name "
    "or a regex for the deliverable.")

   _gDeliverablesCache = {}

   ATTRIB_TYPE_CALLBACK = 0
   ATTRIB_TYPE_REGEX = 1
   ATTRIB_TYPE_VALUE = 2

   def __init__(self, deliverableFile, deliverableClass, config, *args,
    **kwargs):
      object.__init__(self, *args, **kwargs)

      if not os.path.isabs(deliverableFile):
         raise ValueError("Must provide absolute path to Deliverable "
          "constructor")
      elif not os.path.isfile(deliverableFile):
         raise ValueError("Non-existent file passed to Deliverable constructor")
      elif not IsValidDeliverableClass(config, deliverableClass):
         raise ValueError("Non-existent deliverable class passed to "
          "Deliverable constructor: %s" % deliverableClass)

      self.configSection = DeliverableSectionNameFromClass(deliverableClass)
      self.deliverableClass = deliverableClass
      self.file = deliverableFile
      self.regex = None
      self.name = None
      self.filterAttributes = None
      self.attributes = []
      self.attributeHandlers = {}

      deliverableSectionItems = config.GetSectionItems(self.configSection)

      if 'name' in deliverableSectionItems:
         self.name = config.SectionGet(self.configSection, 'name').strip()

      if 'regex' in deliverableSectionItems:
         self.regex = config.SectionGet(self.configSection, 'regex').strip()

      if self.regex is None and self.name is None:
         raise ConfigSpecError(self.ERROR_STR_NEED_NAME_OR_REGEX %
          (deliverableClass))

      if 'attributes' in deliverableSectionItems:
         self.attributes = config.SectionGet(self.configSection, 'attributes',
          list)

      for attr in self.attributes:
         attributeValue = None
         attributeType = None

         if ('attrib_%s_handler' % (attr)) in deliverableSectionItems:
            attributeType = Deliverable.ATTRIB_TYPE_CALLBACK
            attributeValue = config.SectionGet(self.configSection,
             'attrib_%s_handler' % (attr))
         elif ('attrib_%s_regex' % (attr)) in deliverableSectionItems:
            attributeType = Deliverable.ATTRIB_TYPE_REGEX
            attributeValue = config.SectionGet(self.configSection,
             'attrib_%s_regex' % (attr))
         elif ('attrib_%s_value' % (attr)) in deliverableSectionItems:
            attributeType = Deliverable.ATTRIB_TYPE_VALUE
            attributeValue = config.SectionGet(self.configSection,
             'attrib_%s_value' % (attr))
         else:
            raise ConfigSpecError("Deliverable class '%s' defines "
             "attribute '%s', but doesn't define handler for it." %
             (deliverableClass, attr))

         attributeHandlerDescriptor = {}
         attributeHandlerDescriptor['type'] = attributeType

         if attributeType == Deliverable.ATTRIB_TYPE_CALLBACK:
            try:
               attributeHandlerModParts = attributeValue.split('.')
               mod = __import__('.'.join(attributeHandlerModParts[:-1]))
               for comp in attributeHandlerModParts[1:]:
                  mod = getattr(mod, comp)

               attributeHandlerDescriptor['handler'] = mod
            except NameError, ex:
               raise ConfigSpecError("Deliverable class '%s' defines an "
                "attribute callback handler for attribute '%s', but the "
                "callback is undefined: %s" % (self.deliverableClass, attr,
                str(ex)))
         elif (attributeType == Deliverable.ATTRIB_TYPE_REGEX or
          attributeType == Deliverable.ATTRIB_TYPE_VALUE):
            attributeHandlerDescriptor['handler'] = attributeValue
         else:
            assert False, "Unknown attribute handler type: %s" % (attributeType)

         self.attributeHandlers[attr] = attributeHandlerDescriptor

      if 'filter_attributes' in deliverableSectionItems:
         self.filterAttributes = config.SectionGet(self.configSection,
          'filter_attributes', list)

      if self.filterAttributes is not None:
         for fa in self.filterAttributes:
            if fa not in self.attributes:
               raise ConfigSpecError("Deliverable class '%s' defines invalid "
                "filter attribute '%s'" % (deliverableClass, fa))

   def __str__(self):
      return self.GetLocation()

   def GetName(self):
      return self.deliverableClass

   def GetLocation(self):
      return self.file

   def GetFileName(self):
      return os.path.basename(self.GetLocation())

   def GetRegex(self):
      return self.regex

   def GetFilterAttributes(self):
      if self.filterAttributes is None:
         return None
      else:
         return tuple(self.filterAttributes)

   def GetAttributes(self):
      return tuple(self.attributes)

   def GetAttribute(self, attribute):
      if attribute not in self.attributes:
         raise ValueError("Deliverable class '%s' has no attribute '%s" %
          (self.GetName(), attribute))

      handlerType = self.attributeHandlers[attribute]['type']

      if handlerType == Deliverable.ATTRIB_TYPE_VALUE:
         return self.attributeHandlers[attribute]['handler']
      elif handlerType == Deliverable.ATTRIB_TYPE_REGEX:
         attribMatch = re.search(self.attributeHandlers[attribute]['handler'],
          self.GetFileName())

         if attribMatch is None:
            return None
         elif len(attribMatch.groups()) == 1:
            return attribMatch.group(1)
         else:
            return attribMatch.groups()

      elif handlerType == Deliverable.ATTRIB_TYPE_CALLBACK:
         return self.attributeHandlers[attribute]['handler'](self)
      else:
         assert False, "Unknown attribute handler type: %s" % (handlerType)

def FindDeliverables(deliverableDir, config):
   if not os.path.isdir(deliverableDir):
      raise ValueError("Invalid deliverable directory: %s" % (deliverableDir))

   if Deliverable._gDeliverablesCache.has_key(deliverableDir):
      return None

   deliverables = []
   deliverableSections = GetDeliverableSections(config)

   ignoreUndefinedDeliverables = True
   try:
      ignoreUndefinedDeliverables = config.Get('ignore_undefined_deliverables',
       bool)
   except ConfigSpecError, ex:
      if ex.GetDetails() != ConfigSpecError.NO_OPTION_ERROR:
         raise ex

   for root, dirs, files in os.walk(deliverableDir):
      for f in files:
         #print "Looking at: %s" % (os.path.join(root, f))
         deliverableDescList = []
         for section in deliverableSections:
            delivRegex = None
            delivName = None
            matchType = None
            subclassType = None

            sectionItems = config.GetSectionItems(section)

            if 'name' in sectionItems:
               delivName = config.SectionGet(section, 'name').strip()
               matchType = 'name'
            elif 'regex' in sectionItems:
               delivRegex = config.SectionGet(section, 'regex').strip()
               matchType = 'regex'
            else:
               raise ConfigSpecError(Deliverable.ERROR_STR_NEED_NAME_OR_REGEX %
                DeliverableClassFromSectionName(section))

            #print "f is %s, name is %s, regex is %s" % (f, delivName, delivRegex)
            if ((delivName is not None and f == delivName) or
             (delivRegex is not None and re.search(delivRegex, f))):

               if 'subclass' in sectionItems:
                 subclassType = config.SectionGet(section,
                  'subclass').strip()

               delivClassDescription = { 'type': matchType,
                                         'subclass' : subclassType,
                                         'class' : DeliverableClassFromSectionName(section),
                                         'file' : os.path.join(root, f),
               }

               deliverableDescList.append(delivClassDescription)

         if len(deliverableDescList) == 0:
            if not ignoreUndefinedDeliverables:
               assert False, "Should be a release framework error."
            else:
               continue

         if len(deliverableDescList) == 1:
            delivDesc = deliverableDescList[0]

            if delivDesc['subclass'] is not None:
               try:
                  subclassParts = delivDesc['subclass'].split('.')
                  mod = __import__('.'.join(subclassParts[:-1]))
                  for comp in subclassParts[1:]:
                     mod = getattr(mod, comp)

                  newDelivObj = mod(delivDesc['file'], delivDesc['class'],
                   config)

               except NameError, ex:
                  raise ConfigSpecError("subclass error %s" % (ex))
            else:
               newDelivObj = Deliverable(delivDesc['file'],
                delivDesc['class'], config)

            deliverables.append(newDelivObj)

         else:
            matchedClassList = []
            fileLoc = deliverableDescList[0]['file']
            for delivDesc in deliverableDescList:
               assert fileLoc == delivDesc['file'], ("Deliverable file name "
                "mismatch (%s vs %s)?" % (fileLoc, delivDesc['file']))

               matchedClassList.append("%s (matched via %s)" % (
                delivDesc['class'], delivDesc['type']))

            raise ConfigSpecError("More than one deliverable class for "
             "the file %s: %s" % (fileLoc, ', '.join(matchedClassList)))

   Deliverable._gDeliverablesCache[deliverableDir] = tuple(deliverables)
   return len(tuple(deliverables))

def GetAllDeliverables(deliverableDir=None):
   if deliverableDir is not None:
      if not Deliverable._gDeliverablesCache.has_key(deliverableDir):
         raise ValueError("Directory %d has not been scanned for deliverables "
          "yet; use FindDeliverables()" % (deliverableDir))

      return tuple(Deliverable._gDeliverablesCache[deliverableDir])
   else:
      cacheKeys = Deliverable._gDeliverablesCache.keys()
      if len(cacheKeys) == 0:
         raise ValueError("No deliverables found yet; prime cache with "
          "FindDeliverables()" % (deliverableDir))

      allDeliverables = []

      for dDirs in cacheKeys:
         for deliv in Deliverable._gDeliverablesCache[dDirs]:
            allDeliverables.append(deliv)

      return tuple(allDeliverables)

def GetDeliverables(deliverableClass, deliverableDir=None):
   filterArgs = deliverableClass.split(':')
   filterArgsLen = len(filterArgs)
   filteredDeliverableList = []

   # Process the static filters, given in the config file
   for deliv in GetAllDeliverables(deliverableDir):
      staticFilters = deliv.GetName().split(':')
      staticFilterLen = len(staticFilters)

      filterNdx = 0

      skipThisDeliv = False
      while filterNdx < filterArgsLen and filterNdx < staticFilterLen:
         if filterArgs[filterNdx] != staticFilters[filterNdx]:
            skipThisDeliv = True
            break

         filterNdx += 1

      if skipThisDeliv:
         continue

      # If we've parsed all of the filter arguments, we're done
      if filterNdx == filterArgsLen:
         filteredDeliverableList.append(deliv)
         continue

      dynamicFilters = deliv.GetFilterAttributes()

      if dynamicFilters is None and filterNdx < filterArgsLen:
         raise ValueError("GetDeliverables passed filter '%s' for a "
          "deliverable class that defines no filter attributes" %
          ':'.join(filterArgs[filterNdx:]))

      dynamicFilterLen = len(dynamicFilters)

      while filterNdx < filterArgsLen:
         dynNdx = filterNdx - staticFilterLen
         assert dynNdx >= 0, "Invalid (negative) dynamic filter index."
         if dynNdx >= dynamicFilterLen:
            availableFilters = staticFilters[1:] + list(dynamicFilters)
            availableFiltersStr = ', '.join(availableFilters)
            filterCount = len(availableFilters)
            if filterCount > 1:
               pluralFilters = "s"
            else:
               pluralFilters = ""

            raise ValueError("GetDeliverables passed extra filter '%s' "
             "for deliverable %s; %s defines %d filter%s: %s." % (
             ':'.join(filterArgs[filterNdx:]), deliv.GetName(), deliv.GetName(),
             filterCount, pluralFilters, availableFiltersStr))

         if deliv.GetAttribute(dynamicFilters[dynNdx]) != filterArgs[filterNdx]:
            skipThisDeliv = True
            break

         filterNdx += 1

      if skipThisDeliv:
         continue

      filteredDeliverableList.append(deliv)

   return filteredDeliverableList

def GetDeliverable(deliverableClass, deliverableDir=None):
   possibleDelivs = GetDeliverables(deliverableClass, deliverableDir)

   if len(possibleDelivs) == 0:
      return None
   elif len(possibleDelivs) > 1:
      raise ConfigSpecError("More than one deliverable matched for "
       "deliverable class %s: %s" % (deliverableClass,
       ', '.join(possibleDelivs)))
   else:
      return possibleDelivs[0]

def FlushDeliverableCache(deliverableDir=None):
   if deliverableDir is None:
      Deliverable._gDeliverablesCache.clear()
   else:
      try:
         del Deliverable._gDeliverablesCache[deliverableDir]
      except KeyError:
         raise ValueError("Deliverable directory %d not in cache" %
          (deliverableDir))

def GetDeliverableSections(config):
   retSections = []

   for section in config.GetSectionList():
      if IsDeliverableSectionName(section):
         retSections.append(section)

   return retSections

def DeliverableClassFromSectionName(sectionName):
   return re.sub('^%s' % (Deliverable.DELIVERABLE_CONFIG_PREFIX), '',
    sectionName)

def DeliverableSectionNameFromClass(delivClass):
   return Deliverable.DELIVERABLE_CONFIG_PREFIX + delivClass

def IsDeliverableSectionName(sectionName):
   return sectionName.startswith(Deliverable.DELIVERABLE_CONFIG_PREFIX)

def IsValidDeliverableClass(config, delivClass):
   try:
      sectionItems = config.GetSectionItems(DeliverableSectionNameFromClass(
       delivClass))
   except ValueError:
      return False

   return 'name' in sectionItems or 'regex' in sectionItems
<commit_msg>Fix a string format error; when showing which possible deliverables are available, provide the location to the user to help them disambiguate.<commit_after>
import copy
import os
import re
import types

from quickrelease.config import ConfigSpecError, ConfigSpec

class Deliverable(object):
   DELIVERABLE_SECTION_PREFIX = 'deliverable'
   DELIVERABLE_CONFIG_PREFIX = (DELIVERABLE_SECTION_PREFIX +
    ConfigSpec.CONFIG_SECTION_DELIMETER)

   ERROR_STR_NEED_NAME_OR_REGEX = ("Deliverable class '%s' must define a name "
    "or a regex for the deliverable.")

   _gDeliverablesCache = {}

   ATTRIB_TYPE_CALLBACK = 0
   ATTRIB_TYPE_REGEX = 1
   ATTRIB_TYPE_VALUE = 2

   def __init__(self, deliverableFile, deliverableClass, config, *args,
    **kwargs):
      object.__init__(self, *args, **kwargs)

      if not os.path.isabs(deliverableFile):
         raise ValueError("Must provide absolute path to Deliverable "
          "constructor")
      elif not os.path.isfile(deliverableFile):
         raise ValueError("Non-existent file passed to Deliverable constructor")
      elif not IsValidDeliverableClass(config, deliverableClass):
         raise ValueError("Non-existent deliverable class passed to "
          "Deliverable constructor: %s" % deliverableClass)

      self.configSection = DeliverableSectionNameFromClass(deliverableClass)
      self.deliverableClass = deliverableClass
      self.file = deliverableFile
      self.regex = None
      self.name = None
      self.filterAttributes = None
      self.attributes = []
      self.attributeHandlers = {}

      deliverableSectionItems = config.GetSectionItems(self.configSection)

      if 'name' in deliverableSectionItems:
         self.name = config.SectionGet(self.configSection, 'name').strip()

      if 'regex' in deliverableSectionItems:
         self.regex = config.SectionGet(self.configSection, 'regex').strip()

      if self.regex is None and self.name is None:
         raise ConfigSpecError(self.ERROR_STR_NEED_NAME_OR_REGEX %
          (deliverableClass))

      if 'attributes' in deliverableSectionItems:
         self.attributes = config.SectionGet(self.configSection, 'attributes',
          list)

      for attr in self.attributes:
         attributeValue = None
         attributeType = None

         if ('attrib_%s_handler' % (attr)) in deliverableSectionItems:
            attributeType = Deliverable.ATTRIB_TYPE_CALLBACK
            attributeValue = config.SectionGet(self.configSection,
             'attrib_%s_handler' % (attr))
         elif ('attrib_%s_regex' % (attr)) in deliverableSectionItems:
            attributeType = Deliverable.ATTRIB_TYPE_REGEX
            attributeValue = config.SectionGet(self.configSection,
             'attrib_%s_regex' % (attr))
         elif ('attrib_%s_value' % (attr)) in deliverableSectionItems:
            attributeType = Deliverable.ATTRIB_TYPE_VALUE
            attributeValue = config.SectionGet(self.configSection,
             'attrib_%s_value' % (attr))
         else:
            raise ConfigSpecError("Deliverable class '%s' defines "
             "attribute '%s', but doesn't define handler for it." %
             (deliverableClass, attr))

         attributeHandlerDescriptor = {}
         attributeHandlerDescriptor['type'] = attributeType

         if attributeType == Deliverable.ATTRIB_TYPE_CALLBACK:
            try:
               attributeHandlerModParts = attributeValue.split('.')
               mod = __import__('.'.join(attributeHandlerModParts[:-1]))
               for comp in attributeHandlerModParts[1:]:
                  mod = getattr(mod, comp)

               attributeHandlerDescriptor['handler'] = mod
            except NameError, ex:
               raise ConfigSpecError("Deliverable class '%s' defines an "
                "attribute callback handler for attribute '%s', but the "
                "callback is undefined: %s" % (self.deliverableClass, attr,
                str(ex)))
         elif (attributeType == Deliverable.ATTRIB_TYPE_REGEX or
          attributeType == Deliverable.ATTRIB_TYPE_VALUE):
            attributeHandlerDescriptor['handler'] = attributeValue
         else:
            assert False, "Unknown attribute handler type: %s" % (attributeType)

         self.attributeHandlers[attr] = attributeHandlerDescriptor

      if 'filter_attributes' in deliverableSectionItems:
         self.filterAttributes = config.SectionGet(self.configSection,
          'filter_attributes', list)

      if self.filterAttributes is not None:
         for fa in self.filterAttributes:
            if fa not in self.attributes:
               raise ConfigSpecError("Deliverable class '%s' defines invalid "
                "filter attribute '%s'" % (deliverableClass, fa))

   def __str__(self):
      return self.GetLocation()

   def GetName(self):
      return self.deliverableClass

   def GetLocation(self):
      return self.file

   def GetFileName(self):
      return os.path.basename(self.GetLocation())

   def GetRegex(self):
      return self.regex

   def GetFilterAttributes(self):
      if self.filterAttributes is None:
         return None
      else:
         return tuple(self.filterAttributes)

   def GetAttributes(self):
      return tuple(self.attributes)

   def GetAttribute(self, attribute):
      if attribute not in self.attributes:
         raise ValueError("Deliverable class '%s' has no attribute '%s" %
          (self.GetName(), attribute))

      handlerType = self.attributeHandlers[attribute]['type']

      if handlerType == Deliverable.ATTRIB_TYPE_VALUE:
         return self.attributeHandlers[attribute]['handler']
      elif handlerType == Deliverable.ATTRIB_TYPE_REGEX:
         attribMatch = re.search(self.attributeHandlers[attribute]['handler'],
          self.GetFileName())

         if attribMatch is None:
            return None
         elif len(attribMatch.groups()) == 1:
            return attribMatch.group(1)
         else:
            return attribMatch.groups()

      elif handlerType == Deliverable.ATTRIB_TYPE_CALLBACK:
         return self.attributeHandlers[attribute]['handler'](self)
      else:
         assert False, "Unknown attribute handler type: %s" % (handlerType)

def FindDeliverables(deliverableDir, config):
   if not os.path.isdir(deliverableDir):
      raise ValueError("Invalid deliverable directory: %s" % (deliverableDir))

   if Deliverable._gDeliverablesCache.has_key(deliverableDir):
      return None

   deliverables = []
   deliverableSections = GetDeliverableSections(config)

   ignoreUndefinedDeliverables = True
   try:
      ignoreUndefinedDeliverables = config.Get('ignore_undefined_deliverables',
       bool)
   except ConfigSpecError, ex:
      if ex.GetDetails() != ConfigSpecError.NO_OPTION_ERROR:
         raise ex

   for root, dirs, files in os.walk(deliverableDir):
      for f in files:
         #print "Looking at: %s" % (os.path.join(root, f))
         deliverableDescList = []
         for section in deliverableSections:
            delivRegex = None
            delivName = None
            matchType = None
            subclassType = None

            sectionItems = config.GetSectionItems(section)

            if 'name' in sectionItems:
               delivName = config.SectionGet(section, 'name').strip()
               matchType = 'name'
            elif 'regex' in sectionItems:
               delivRegex = config.SectionGet(section, 'regex').strip()
               matchType = 'regex'
            else:
               raise ConfigSpecError(Deliverable.ERROR_STR_NEED_NAME_OR_REGEX %
                DeliverableClassFromSectionName(section))

            #print "f is %s, name is %s, regex is %s" % (f, delivName, delivRegex)
            if ((delivName is not None and f == delivName) or
             (delivRegex is not None and re.search(delivRegex, f))):

               if 'subclass' in sectionItems:
                 subclassType = config.SectionGet(section,
                  'subclass').strip()

               delivClassDescription = { 'type': matchType,
                                         'subclass' : subclassType,
                                         'class' : DeliverableClassFromSectionName(section),
                                         'file' : os.path.join(root, f),
               }

               deliverableDescList.append(delivClassDescription)

         if len(deliverableDescList) == 0:
            if not ignoreUndefinedDeliverables:
               assert False, "Should be a release framework error."
            else:
               continue

         if len(deliverableDescList) == 1:
            delivDesc = deliverableDescList[0]

            if delivDesc['subclass'] is not None:
               try:
                  subclassParts = delivDesc['subclass'].split('.')
                  mod = __import__('.'.join(subclassParts[:-1]))
                  for comp in subclassParts[1:]:
                     mod = getattr(mod, comp)

                  newDelivObj = mod(delivDesc['file'], delivDesc['class'],
                   config)

               except NameError, ex:
                  raise ConfigSpecError("subclass error %s" % (ex))
            else:
               newDelivObj = Deliverable(delivDesc['file'],
                delivDesc['class'], config)

            deliverables.append(newDelivObj)

         else:
            matchedClassList = []
            fileLoc = deliverableDescList[0]['file']
            for delivDesc in deliverableDescList:
               assert fileLoc == delivDesc['file'], ("Deliverable file name "
                "mismatch (%s vs %s)?" % (fileLoc, delivDesc['file']))

               matchedClassList.append("%s (matched via %s)" % (
                delivDesc['class'], delivDesc['type']))

            raise ConfigSpecError("More than one deliverable class for "
             "the file %s: %s" % (fileLoc, ', '.join(matchedClassList)))

   Deliverable._gDeliverablesCache[deliverableDir] = tuple(deliverables)
   return len(tuple(deliverables))

def GetAllDeliverables(deliverableDir=None):
   if deliverableDir is not None:
      if not Deliverable._gDeliverablesCache.has_key(deliverableDir):
         raise ValueError("Directory %d has not been scanned for deliverables "
          "yet; use FindDeliverables()" % (deliverableDir))

      return tuple(Deliverable._gDeliverablesCache[deliverableDir])
   else:
      cacheKeys = Deliverable._gDeliverablesCache.keys()
      if len(cacheKeys) == 0:
         raise ValueError("No deliverables found yet; prime cache with "
          "FindDeliverables()")

      allDeliverables = []

      for dDirs in cacheKeys:
         for deliv in Deliverable._gDeliverablesCache[dDirs]:
            allDeliverables.append(deliv)

      return tuple(allDeliverables)

def GetDeliverables(deliverableClass, deliverableDir=None):
   filterArgs = deliverableClass.split(':')
   filterArgsLen = len(filterArgs)
   filteredDeliverableList = []

   # Process the static filters, given in the config file
   for deliv in GetAllDeliverables(deliverableDir):
      staticFilters = deliv.GetName().split(':')
      staticFilterLen = len(staticFilters)

      filterNdx = 0

      skipThisDeliv = False
      while filterNdx < filterArgsLen and filterNdx < staticFilterLen:
         if filterArgs[filterNdx] != staticFilters[filterNdx]:
            skipThisDeliv = True
            break

         filterNdx += 1

      if skipThisDeliv:
         continue

      # If we've parsed all of the filter arguments, we're done
      if filterNdx == filterArgsLen:
         filteredDeliverableList.append(deliv)
         continue

      dynamicFilters = deliv.GetFilterAttributes()

      if dynamicFilters is None and filterNdx < filterArgsLen:
         raise ValueError("GetDeliverables passed filter '%s' for a "
          "deliverable class that defines no filter attributes" %
          ':'.join(filterArgs[filterNdx:]))

      dynamicFilterLen = len(dynamicFilters)

      while filterNdx < filterArgsLen:
         dynNdx = filterNdx - staticFilterLen
         assert dynNdx >= 0, "Invalid (negative) dynamic filter index."
         if dynNdx >= dynamicFilterLen:
            availableFilters = staticFilters[1:] + list(dynamicFilters)
            availableFiltersStr = ', '.join(availableFilters)
            filterCount = len(availableFilters)
            if filterCount > 1:
               pluralFilters = "s"
            else:
               pluralFilters = ""

            raise ValueError("GetDeliverables passed extra filter '%s' "
             "for deliverable %s; %s defines %d filter%s: %s." % (
             ':'.join(filterArgs[filterNdx:]), deliv.GetName(), deliv.GetName(),
             filterCount, pluralFilters, availableFiltersStr))

         if deliv.GetAttribute(dynamicFilters[dynNdx]) != filterArgs[filterNdx]:
            skipThisDeliv = True
            break

         filterNdx += 1

      if skipThisDeliv:
         continue

      filteredDeliverableList.append(deliv)

   return filteredDeliverableList

def GetDeliverable(deliverableClass, deliverableDir=None):
   possibleDelivs = GetDeliverables(deliverableClass, deliverableDir)

   if len(possibleDelivs) == 0:
      return None
   elif len(possibleDelivs) > 1:
      raise ConfigSpecError("More than one deliverable matched for "
       "deliverable class %s: %s" % (deliverableClass,
       ', '.join(list(x.GetLocation() for x in possibleDelivs))))
   else:
      return possibleDelivs[0]

def FlushDeliverableCache(deliverableDir=None):
   if deliverableDir is None:
      Deliverable._gDeliverablesCache.clear()
   else:
      try:
         del Deliverable._gDeliverablesCache[deliverableDir]
      except KeyError:
         raise ValueError("Deliverable directory %d not in cache" %
          (deliverableDir))

def GetDeliverableSections(config):
   retSections = []

   for section in config.GetSectionList():
      if IsDeliverableSectionName(section):
         retSections.append(section)

   return retSections

def DeliverableClassFromSectionName(sectionName):
   return re.sub('^%s' % (Deliverable.DELIVERABLE_CONFIG_PREFIX), '',
    sectionName)

def DeliverableSectionNameFromClass(delivClass):
   return Deliverable.DELIVERABLE_CONFIG_PREFIX + delivClass

def IsDeliverableSectionName(sectionName):
   return sectionName.startswith(Deliverable.DELIVERABLE_CONFIG_PREFIX)

def IsValidDeliverableClass(config, delivClass):
   try:
      sectionItems = config.GetSectionItems(DeliverableSectionNameFromClass(
       delivClass))
   except ValueError:
      return False

   return 'name' in sectionItems or 'regex' in sectionItems
<|endoftext|>


<commit_before>from .models import *
from django.contrib import admin

class GenericMetaPropInline(admin.TabularInline):
    model = GenericMetaProp
    # readonly_fields = ('slug','created')
    #fields = ('code','namespace')
    # related_search_fields = {'label' : ('name','slug')}
    extra=1

class GenericMetaPropAdmin(admin.ModelAdmin):
    pass

class ObjectTypeAdmin(admin.ModelAdmin):
    pass

class AttributeMappingInline(admin.TabularInline):
    model = AttributeMapping
    # readonly_fields = ('slug','created')
    #fields = ('code','namespace')
    # related_search_fields = {'label' : ('name','slug')}
    extra=1

class EmbeddedMappingInline(admin.TabularInline):
    model = EmbeddedMapping
    # readonly_fields = ('slug','created')
    #fields = ('code','namespace')
    # related_search_fields = {'label' : ('name','slug')}
    extra=1

class ChainedMappingInline(admin.TabularInline):
    model = ChainedMapping
    fk_name='scope'
    # readonly_fields = ('slug','created')
    fields = ('attr','predicate','chainedMapping')
    # related_search_fields = {'label' : ('name','slug')}
    extra=1

class ObjectMappingAdmin(admin.ModelAdmin):
    search_fields = ['content_type__name' ]
    inlines = [   AttributeMappingInline, ChainedMappingInline, EmbeddedMappingInline]
    filter_horizontal = ('obj_type',)
    pass

class AttributeMappingAdmin(admin.ModelAdmin):
    pass

class EmbeddedMappingAdmin(admin.ModelAdmin):
    pass

class NamespaceAdmin(admin.ModelAdmin):
    list_display = ('uri','prefix','notes')
    fields = ('uri','prefix','notes')
#    related_search_fields = {'concept' : ('pref_label','definition')}
    #list_editable = ('name','slug')
    search_fields = ['uri','prefix']

class ConfigVarAdmin(admin.ModelAdmin):
    pass

class ImportedResourceAdmin(admin.ModelAdmin):
#    list_display = ('file', 'remote', 'resource_type')
    search_fields = ['description','file','remote']
    pass

class ObjectBoundListFilter(admin.SimpleListFilter):
    title='Chain Start by Object Type'
    parameter_name = 'objtype'

    def lookups(self, request, model_admin):
        chains = ServiceBinding.objects.filter(object_mapping__content_type__isnull=False)
        return set([(c.object_mapping.first().content_type.model, c.object_mapping.first().content_type.model) for c in chains])

    def queryset(self, request, qs):
        try:
            #import pdb; pdb.set_trace()
            if request.GET.get('objtype') :
                qs= qs.filter(object_mapping__content_type__model = request.GET.get('objtype'))
        except:
            pass
        return qs

class ChainListFilter(admin.SimpleListFilter):
    title='Chain members'
    parameter_name = 'chain_id'

    def lookups(self, request, model_admin):
        chains = ServiceBinding.objects.filter(object_mapping__isnull=False)
        return [(c.id, c.object_mapping.first().name) for c in chains]

    def queryset(self, request, qs):
        try:
            pass
            #qs= qs.filter(object_mapping__id = request.GET.get('chain_id'))
        except:
            pass
        return qs

class ServiceBindingAdmin(admin.ModelAdmin) :
    list_display = ('title', 'binding_type', 'next_service')
    list_filter=(ObjectBoundListFilter,ChainListFilter,'binding_type')
    search_fields = ['title','binding_type']
    pass

admin.site.register(Namespace, NamespaceAdmin)
admin.site.register(GenericMetaProp,GenericMetaPropAdmin)
admin.site.register(ObjectType, ObjectTypeAdmin)
admin.site.register(ObjectMapping, ObjectMappingAdmin)
#admin.site.register(AttributeMapping, AttributeMappingAdmin)
#admin.site.register(EmbeddedMapping, EmbeddedMappingAdmin)
admin.site.register(ImportedResource, ImportedResourceAdmin)

admin.site.register(ServiceBinding, ServiceBindingAdmin)
admin.site.register(ConfigVar, ConfigVarAdmin)<commit_msg>handling missing name from object mapping.<commit_after>from .models import *
from django.contrib import admin

class GenericMetaPropInline(admin.TabularInline):
    model = GenericMetaProp
    # readonly_fields = ('slug','created')
    #fields = ('code','namespace')
    # related_search_fields = {'label' : ('name','slug')}
    extra=1

class GenericMetaPropAdmin(admin.ModelAdmin):
    pass

class ObjectTypeAdmin(admin.ModelAdmin):
    pass

class AttributeMappingInline(admin.TabularInline):
    model = AttributeMapping
    # readonly_fields = ('slug','created')
    #fields = ('code','namespace')
    # related_search_fields = {'label' : ('name','slug')}
    extra=1

class EmbeddedMappingInline(admin.TabularInline):
    model = EmbeddedMapping
    # readonly_fields = ('slug','created')
    #fields = ('code','namespace')
    # related_search_fields = {'label' : ('name','slug')}
    extra=1

class ChainedMappingInline(admin.TabularInline):
    model = ChainedMapping
    fk_name='scope'
    # readonly_fields = ('slug','created')
    fields = ('attr','predicate','chainedMapping')
    # related_search_fields = {'label' : ('name','slug')}
    extra=1

class ObjectMappingAdmin(admin.ModelAdmin):
    search_fields = ['content_type__name' ]
    inlines = [   AttributeMappingInline, ChainedMappingInline, EmbeddedMappingInline]
    filter_horizontal = ('obj_type',)
    pass

class AttributeMappingAdmin(admin.ModelAdmin):
    pass

class EmbeddedMappingAdmin(admin.ModelAdmin):
    pass

class NamespaceAdmin(admin.ModelAdmin):
    list_display = ('uri','prefix','notes')
    fields = ('uri','prefix','notes')
#    related_search_fields = {'concept' : ('pref_label','definition')}
    #list_editable = ('name','slug')
    search_fields = ['uri','prefix']

class ConfigVarAdmin(admin.ModelAdmin):
    pass

class ImportedResourceAdmin(admin.ModelAdmin):
#    list_display = ('file', 'remote', 'resource_type')
    search_fields = ['description','file','remote']
    pass

class ObjectBoundListFilter(admin.SimpleListFilter):
    title='Chain Start by Object Type'
    parameter_name = 'objtype'

    def lookups(self, request, model_admin):
        chains = ServiceBinding.objects.filter(object_mapping__content_type__isnull=False)
        return set([(c.object_mapping.first().content_type.model, c.object_mapping.first().content_type.model) for c in chains])

    def queryset(self, request, qs):
        try:
            #import pdb; pdb.set_trace()
            if request.GET.get('objtype') :
                qs= qs.filter(object_mapping__content_type__model = request.GET.get('objtype'))
        except:
            pass
        return qs

class ChainListFilter(admin.SimpleListFilter):
    title='Chain members'
    parameter_name = 'chain_id'

    def lookups(self, request, model_admin):
        chains = ServiceBinding.objects.filter(object_mapping__name__isnull=False)
        return [(c.id, c.object_mapping.first().name) for c in chains]

    def queryset(self, request, qs):
        try:
            pass
            #qs= qs.filter(object_mapping__id = request.GET.get('chain_id'))
        except:
            pass
        return qs

class ServiceBindingAdmin(admin.ModelAdmin) :
    list_display = ('title', 'binding_type', 'next_service')
    list_filter=(ObjectBoundListFilter,ChainListFilter,'binding_type')
    search_fields = ['title','binding_type']
    pass

admin.site.register(Namespace, NamespaceAdmin)
admin.site.register(GenericMetaProp,GenericMetaPropAdmin)
admin.site.register(ObjectType, ObjectTypeAdmin)
admin.site.register(ObjectMapping, ObjectMappingAdmin)
#admin.site.register(AttributeMapping, AttributeMappingAdmin)
#admin.site.register(EmbeddedMapping, EmbeddedMappingAdmin)
admin.site.register(ImportedResource, ImportedResourceAdmin)

admin.site.register(ServiceBinding, ServiceBindingAdmin)
admin.site.register(ConfigVar, ConfigVarAdmin)<|endoftext|>


<commit_before>"""
Created on 12 Jul 2016

@author: ikalvari

TO DO:   - Rename functions
         - Split fetch_gca_data
"""

# ---------------------------------IMPORTS-------------------------------------

import os
import sys
import datetime
import xml.etree.ElementTree as ET
import requests

# Config Files
import genome_fetch as gf
from config import gen_config as gc

# -----------------------------------------------------------------------------

def fetch_gca_data(upid, assembly_acc, kingdom):
    """
    Parses ENA GCA accession xml, and returns the accession's data in the
    form of a dictionary

    upid: A valid Uniprot proteome accession (e.g. UP000005640 - Homo Sapiens)
    assembly_acc: A valid ENA GCA accession
    kingdom: The corresponding species kingdom
    """

    genome_entry = {}
    fields = {}
    tmp_acc = assembly_acc

    '''
    if tmp_acc.find('.') != -1:
        tmp_acc = tmp_acc.partition('.')[0]
    '''

    assembly_xml = requests.get(gc.ENA_XML_URL % tmp_acc).content

    root = ET.fromstring(assembly_xml)
    assembly = root.find("ASSEMBLY")

    primary_id = assembly.find("IDENTIFIERS").find("PRIMARY_ID")

    if primary_id is not None:
        fields["gca_acc"] = primary_id.text
    else:
        fields["gca_acc"] = primary_id

    version = fields["gca_acc"].partition('.')[2]
    fields["gca_version"] = int(version)

    fields["ensembl_id"] = None  # add these as a post-processing step
    fields["assembly_name"] = assembly.find("NAME").text

    ass_links = None
    ass_level = assembly.find("ASSEMBLY_LEVEL").text
    ass_links = assembly.find("ASSEMBLY_LINKS")

    fields["assembly_level"] = ass_level

    if ass_level == "contig" and ass_links is None:
        wgs_fields = assembly.find("WGS_SET")
        wgs_acc = gf.get_wgs_set_accession(
            wgs_fields.find("PREFIX").text, wgs_fields.find("VERSION").text)

        fields["wgs_acc"] = wgs_acc
        fields["wgs_version"] = int(wgs_fields.find("VERSION").text)
    else:
        fields["wgs_acc"] = None
        fields["wgs_version"] = None

    fields["study_ref"] = assembly.find("STUDY_REF").find(
        "IDENTIFIERS").find("PRIMARY_ID").text

    # description can be very long, using title instead as short description
    fields["description"] = assembly.find("TITLE").text

    attributes = fetch_assembly_attributes(
        assembly.find("ASSEMBLY_ATTRIBUTES"))

    fields["total_length"] = int(attributes["total-length"])
    fields["ungapped_length"] = int(attributes["ungapped-length"])

    genome_desc = assembly.find("DESCRIPTION").text

    if genome_desc is not None:
        if genome_desc.find("circular") != -1:
            fields["circular"] = 1
        else:
            fields["circular"] = 0
    else:
        fields["circular"] = None

    taxid = assembly.find("TAXON")
    fields["ncbi_id"] = int(taxid.find("TAXON_ID").text)

    fields["scientific_name"] = taxid.find("SCIENTIFIC_NAME").text

    common_name = None
    common_name = taxid.find("COMMON_NAME")
    if common_name is not None:
        fields["common_name"] = taxid.find("COMMON_NAME").text
    fields["kingdom"] = kingdom

    fields["num_gen_regions"] = int(attributes["count-regions"])
    fields["num_rfam_regions"] = None
    fields["num_families"] = None

    # this takes the date of the entry is created
    entry_date = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
    fields["created"] = entry_date
    fields["updated"] = entry_date

    genome_entry["model"] = gc.GENOME_MODEL
    genome_entry["pk"] = upid
    genome_entry["fields"] = fields

    return genome_entry

# -----------------------------------------------------------------------------

def fetch_assembly_accessions(upid, gca_acc, acc_ftp_link, reg_ftp_link=None):
    """
    Parses assembly report file and exports all assembly accessions in a
    django dict format..

    upid: A valid Uniprot proteome accession (e.g. UP000005640 - Homo Sapiens)
    gca_acc: A valid ENA GCA accession
    acc_ftp_link: Assembly report file ftp url (as retrieved from GCA xml file)
    reg_ftp_link: Assembly region file ftp url (as retrieved from GCA xml file)
    """

    assembly_accs = []
    fields = {}
    entry = {}
    regions = None

    # load accession regions
    if reg_ftp_link is not None:
        regions = region_loader(reg_ftp_link)

    http_link = acc_ftp_link.replace("ftp://", "http://")
    response = requests.get(http_link).content

    accessions = response.strip().split('\n')
    # remove header
    accessions.pop(0)

    accession = ''
    for acc_line in accessions:
        accession = acc_line.strip().split('\t')

        if (accession[0].find('.') != -1):
            entry["model"] = gc.GENSEQ_MODEL
            entry["pk"] = str(accession[0])  # seq_acc

            if len(accession) == 7:
                acc_meta = fetch_gca_acc_metadata(accession[0])

                fields["ncbi_id"] = acc_meta["ncbi_id"]
                fields["description"] = acc_meta["description"]
                fields["upid"] = upid
                fields["gen_acc"] = gca_acc
                fields["seq_version"] = int(accession[0].partition('.')[2])
                fields["seq_length"] = int(accession[2])
                fields["seq_role"] = accession[3]  # patch, loci etc

                # need to parse the regions file for these fields
                if reg_ftp_link is not None and (accession[0] in regions.keys()):
                    fields["seq_start"] = int(regions[accession[0]][0])
                    fields["seq_end"] = int(regions[accession[0]][1])
                else:
                    fields["seq_start"] = 0
                    fields["seq_end"] = 0

                fields["mol_type"] = acc_meta["mol_type"]
                # primary, PATCHES, ALT_REF_LOCI_1
                fields["assembly_unit"] = accession[6]
                # this takes the date of the entry is created
                entry_date = datetime.datetime.now().strftime(
                    "%Y-%m-%d %H:%M:%S")
                fields["created"] = entry_date
                fields["updated"] = entry_date

            elif(len(accession) < 7 and accession[0].find('.') != -1):
                fields = fetch_wgs_acc_metadata(accession[0])

        else:
            # skip cases no accession is provided
            pass

        entry["fields"] = fields
        assembly_accs.append(entry)
        fields = {}
        entry = {}

    return assembly_accs

# -----------------------------------------------------------------------------

def region_loader(reg_ftp_link):
    """
    Parses an assembly's region file and builds a dictionary of accessions and
    start-end coordinates which are stored in a tuple format

    reg_ftp_link: The ftp url of the corresponding region file
    """

    region_dict = {}

    http_link = reg_ftp_link.replace("ftp://", "http://")
    response = requests.get(http_link).content

    regions = response.strip().split('\n')

    # remove header
    regions.pop(0)

    for region in regions:
        region = region.strip().split('\t')
        coords = region[5].split('-')
        region_dict[region[1]] = (int(coords[0]), int(coords[1]))

    return region_dict

# -----------------------------------------------------------------------------

def fetch_wgs_metadata(upid, assembly_acc, kingdom):
    """
    Parses ENA WGS accession xml, and returns the accession's data in the
    form of a dictionary

    upid: A valid Uniprot's proteome id
    assembly_acc: A valid ENA's WGS accession
    kingdom: A string representing the kingdom a species belongs to (e.g.viruses)
    """

    wgs_entry = {}
    fields = {}

    response = requests.get(gc.ENA_XML_URL % assembly_acc).content
    assembly_xml = ET.fromstring(response)  # root
    entry = assembly_xml.find("entry")

    fields["gca_acc"] = None
    fields["gca_version"] = None

    fields["wgs_acc"] = entry.get("accession")
    fields["wgs_version"] = int(entry.get("version"))
    fields["ensembl_id"] = None  # post processing

    # fetching assembly name out of entry's comment lines

    entry_cmnt = None
    entry_cmnt = entry.find("comment")

    if entry_cmnt is not None:
        assembly_name = get_wgs_assembly_name(
            entry.find("comment").text)
        fields["assembly_name"] = assembly_name

    else:
        fields["assembly_name"] = None

    if entry.get("topology") == "linear":
        fields["circular"] = 0
    else:
        fields["circular"] = 1

    fields["assembly_level"] = "contig"

    fields["study_ref"] = entry.find("projectAccession").text
    fields["description"] = entry.find("reference").find("title").text
    fields["kingdom"] = kingdom

    taxon = entry.find("feature").find("taxon")
    fields["scientific_name"] = taxon.get("scientificName")
    fields["common_name"] = None
    fields["ncbi_id"] = int(taxon.get("taxId"))

    fields["total_length"] = None
    fields["ungapped_length"] = None

    fields["num_gen_regions"] = None

    fields["num_rfam_regions"] = None  # post process
    fields["num_families"] = None  # post process

    # this takes the date of the entry is created
    entry_date = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
    fields["created"] = entry_date
    fields["updated"] = entry_date

    wgs_entry["model"] = gc.GENOME_MODEL
    wgs_entry["pk"] = upid
    wgs_entry["fields"] = fields

    return wgs_entry

# -----------------------------------------------------------------------------

def fetch_wgs_accs_metadata(upid, assembly_acc, wgs_range):
    """
    This function runs over the wgs range and exports all metadata for every
    accession in the provided wgs range

    wgs_range: A valid ENA-WGS set range
    """

    wgs_entries = []
    entry = {}
    fields = {}
    wgs_accs = gf.fetch_wgs_range_accs(wgs_range)

    for acc in wgs_accs:
        entry["model"] = gc.GENSEQ_MODEL
        entry["pk"] = acc
        fields = fetch_wgs_acc_metadata(acc)
        entry["fields"] = fields
        # adding upid and wgs_acc in entry fields
        entry["fields"]["upid"] = upid
        entry["fields"]["gen_acc"] = assembly_acc
        wgs_entries.append(entry)
        entry = {}

    return wgs_entries

# -----------------------------------------------------------------------------

def fetch_wgs_acc_metadata(wgs_acc):
    """
    Return a fields dictionary

    wgs_acc: A valid ENA wgs accession
    """

    fields = {}
    response = requests.get(gf.ENA_XML_URL % wgs_acc).content
    acc_xml = ET.fromstring(response)
    entry = acc_xml.find("entry")

    fields["seq_version"] = int(entry.get("entryVersion"))
    fields["seq_length"] = int(entry.get("sequenceLength"))
    fields["seq_role"] = "contig"
    fields["mol_type"] = entry.get("moleculeType")

    taxon_node = None
    taxon_node = entry.find("feature").find("taxon")
    fields["ncbi_id"] = int(taxon_node.get("taxId"))

    location = None
    refs = None
    refs = entry.findall("reference")

    for ref in refs:
        location = ref.get("location")
        if location is not None:
            break

    if location is not None:
        location = location.strip().split('-')
        fields["seq_start"] = int(location[0])
        fields["seq_end"] = int(location[1])
    else:
        fields["seq_start"] = 0
        fields["seq_end"] = 0

    fields["assembly_unit"] = None  # need to check this one
    fields["description"] = entry.find("description").text

    # this takes the date of the entry is created
    entry_date = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
    fields["created"] = entry_date
    fields["updated"] = entry_date  # when adding a new genome

    return fields
# -----------------------------------------------------------------------------

def get_wgs_assembly_name(wgs_comment):
    """
    Search the wgs comment text and return the Assembly name

    wgs_comment: WGS comment text
    """

    assembly_name = ''
    cmnt_lines = wgs_comment.strip().split('\n')

    for line in cmnt_lines:
        if line.find("Assembly Name") != -1:
            assembly_name = line
            break

    assembly_name = assembly_name.strip().partition("::")[2].strip()

    return assembly_name
# -----------------------------------------------------------------------------

def fetch_assembly_attributes(attrs_node):
    """
    Runs over the attributes node in the xml file and returns all attribute
    tag-value pairs in a dictionary format

    attrs_node: GCA xml's attributes node
    """

    attribute_values = {}
    attributes = attrs_node.findall("ASSEMBLY_ATTRIBUTE")

    for attr in attributes:
        tag = attr.find("TAG").text
        value = attr.find("VALUE").text
        attribute_values[tag] = value

    return attribute_values

# -----------------------------------------------------------------------------

def fetch_gca_acc_metadata(accession):
    """
    Fetch accession metadata and return a dictionary with ncbi_id, molecule's
    type, description

    accession: A valid GCA related ENA accession
    """

    metadata = {}
    xml_str = requests.get(gc.ENA_XML_URL % accession).content

    xml_root = ET.fromstring(xml_str)

    entry = xml_root.find("entry")

    '''
    mol_type = None
    mol_type = entry.get("moleculeType")
    '''

    # None if no moleculeType found
    metadata["mol_type"] = entry.get("moleculeType")
    metadata["description"] = entry.find("description").text
    metadata["ncbi_id"] = int(entry.find("feature").find("taxon").get("taxId"))

    return metadata

# -----------------------------------------------------------------------------

def fetch_assembly_links(gca_acc):
    """
    Retrieves and returns a dictionary with all ftp links found in the GCA xml
    file.

    gca_acc: A valid ENA GCA accession
    """

    gca_ftp_links = {}

    response = requests.get(gc.ENA_XML_URL % gca_acc).content

    xml = ET.fromstring(response)

    # fetch assembly links node
    assembly_links = xml.find("ASSEMBLY").find(
        "ASSEMBLY_LINKS").findall("ASSEMBLY_LINK")

    # loop over all available links
    for link in assembly_links:
        label = link.find("URL_LINK").find("LABEL").text.replace(' ', '_')
        url = link.find("URL_LINK").find("URL").text

        gca_ftp_links[label] = url

        label = None
        url = None

    return gca_ftp_links

# -----------------------------------------------------------------------------
if __name__ == '__main__':
    pass
<commit_msg>added ensembl_source field - fetch_gca_data<commit_after>"""
Created on 12 Jul 2016

@author: ikalvari

TO DO:   - Rename functions
         - Split fetch_gca_data
"""

# ---------------------------------IMPORTS-------------------------------------

import os
import sys
import datetime
import xml.etree.ElementTree as ET
import requests

# Config Files
import genome_fetch as gf
from config import gen_config as gc

# -----------------------------------------------------------------------------

def fetch_gca_data(upid, assembly_acc, kingdom):
    """
    Parses ENA GCA accession xml, and returns the accession's data in the
    form of a dictionary

    upid: A valid Uniprot proteome accession (e.g. UP000005640 - Homo Sapiens)
    assembly_acc: A valid ENA GCA accession
    kingdom: The corresponding species kingdom
    """

    genome_entry = {}
    fields = {}
    tmp_acc = assembly_acc

    '''
    if tmp_acc.find('.') != -1:
        tmp_acc = tmp_acc.partition('.')[0]
    '''

    assembly_xml = requests.get(gc.ENA_XML_URL % tmp_acc).content

    root = ET.fromstring(assembly_xml)
    assembly = root.find("ASSEMBLY")

    primary_id = assembly.find("IDENTIFIERS").find("PRIMARY_ID")

    if primary_id is not None:
        fields["gca_acc"] = primary_id.text
    else:
        fields["gca_acc"] = primary_id

    version = fields["gca_acc"].partition('.')[2]
    fields["gca_version"] = int(version)

    # add ensembl fields as a post-processing step
    fields["ensembl_id"] = None
    fields["ensembl_source"] = None
    fields["assembly_name"] = assembly.find("NAME").text

    ass_links = None
    ass_level = assembly.find("ASSEMBLY_LEVEL").text
    ass_links = assembly.find("ASSEMBLY_LINKS")

    fields["assembly_level"] = ass_level

    if ass_level == "contig" and ass_links is None:
        wgs_fields = assembly.find("WGS_SET")
        wgs_acc = gf.get_wgs_set_accession(
            wgs_fields.find("PREFIX").text, wgs_fields.find("VERSION").text)

        fields["wgs_acc"] = wgs_acc
        fields["wgs_version"] = int(wgs_fields.find("VERSION").text)
    else:
        fields["wgs_acc"] = None
        fields["wgs_version"] = None

    fields["study_ref"] = assembly.find("STUDY_REF").find(
        "IDENTIFIERS").find("PRIMARY_ID").text

    # description can be very long, using title instead as short description
    fields["description"] = assembly.find("TITLE").text

    attributes = fetch_assembly_attributes(
        assembly.find("ASSEMBLY_ATTRIBUTES"))

    fields["total_length"] = int(attributes["total-length"])
    fields["ungapped_length"] = int(attributes["ungapped-length"])

    genome_desc = assembly.find("DESCRIPTION").text

    if genome_desc is not None:
        if genome_desc.find("circular") != -1:
            fields["circular"] = 1
        else:
            fields["circular"] = 0
    else:
        fields["circular"] = None

    taxid = assembly.find("TAXON")
    fields["ncbi_id"] = int(taxid.find("TAXON_ID").text)

    fields["scientific_name"] = taxid.find("SCIENTIFIC_NAME").text

    common_name = None
    common_name = taxid.find("COMMON_NAME")
    if common_name is not None:
        fields["common_name"] = taxid.find("COMMON_NAME").text
    fields["kingdom"] = kingdom

    fields["num_gen_regions"] = int(attributes["count-regions"])
    fields["num_rfam_regions"] = None
    fields["num_families"] = None

    # this takes the date of the entry is created
    entry_date = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
    fields["created"] = entry_date
    fields["updated"] = entry_date

    genome_entry["model"] = gc.GENOME_MODEL
    genome_entry["pk"] = upid
    genome_entry["fields"] = fields

    return genome_entry

# -----------------------------------------------------------------------------

def fetch_assembly_accessions(upid, gca_acc, acc_ftp_link, reg_ftp_link=None):
    """
    Parses assembly report file and exports all assembly accessions in a
    django dict format..

    upid: A valid Uniprot proteome accession (e.g. UP000005640 - Homo Sapiens)
    gca_acc: A valid ENA GCA accession
    acc_ftp_link: Assembly report file ftp url (as retrieved from GCA xml file)
    reg_ftp_link: Assembly region file ftp url (as retrieved from GCA xml file)
    """

    assembly_accs = []
    fields = {}
    entry = {}
    regions = None

    # load accession regions
    if reg_ftp_link is not None:
        regions = region_loader(reg_ftp_link)

    http_link = acc_ftp_link.replace("ftp://", "http://")
    response = requests.get(http_link).content

    accessions = response.strip().split('\n')
    # remove header
    accessions.pop(0)

    accession = ''
    for acc_line in accessions:
        accession = acc_line.strip().split('\t')

        if (accession[0].find('.') != -1):
            entry["model"] = gc.GENSEQ_MODEL
            entry["pk"] = str(accession[0])  # seq_acc

            if len(accession) == 7:
                acc_meta = fetch_gca_acc_metadata(accession[0])

                fields["ncbi_id"] = acc_meta["ncbi_id"]
                fields["description"] = acc_meta["description"]
                fields["upid"] = upid
                fields["gen_acc"] = gca_acc
                fields["seq_version"] = int(accession[0].partition('.')[2])
                fields["seq_length"] = int(accession[2])
                fields["seq_role"] = accession[3]  # patch, loci etc

                # need to parse the regions file for these fields
                if reg_ftp_link is not None and (accession[0] in regions.keys()):
                    fields["seq_start"] = int(regions[accession[0]][0])
                    fields["seq_end"] = int(regions[accession[0]][1])
                else:
                    fields["seq_start"] = 0
                    fields["seq_end"] = 0

                fields["mol_type"] = acc_meta["mol_type"]
                # primary, PATCHES, ALT_REF_LOCI_1
                fields["assembly_unit"] = accession[6]
                # this takes the date of the entry is created
                entry_date = datetime.datetime.now().strftime(
                    "%Y-%m-%d %H:%M:%S")
                fields["created"] = entry_date
                fields["updated"] = entry_date

            elif(len(accession) < 7 and accession[0].find('.') != -1):
                fields = fetch_wgs_acc_metadata(accession[0])

        else:
            # skip cases no accession is provided
            pass

        entry["fields"] = fields
        assembly_accs.append(entry)
        fields = {}
        entry = {}

    return assembly_accs

# -----------------------------------------------------------------------------

def region_loader(reg_ftp_link):
    """
    Parses an assembly's region file and builds a dictionary of accessions and
    start-end coordinates which are stored in a tuple format

    reg_ftp_link: The ftp url of the corresponding region file
    """

    region_dict = {}

    http_link = reg_ftp_link.replace("ftp://", "http://")
    response = requests.get(http_link).content

    regions = response.strip().split('\n')

    # remove header
    regions.pop(0)

    for region in regions:
        region = region.strip().split('\t')
        coords = region[5].split('-')
        region_dict[region[1]] = (int(coords[0]), int(coords[1]))

    return region_dict

# -----------------------------------------------------------------------------

def fetch_wgs_metadata(upid, assembly_acc, kingdom):
    """
    Parses ENA WGS accession xml, and returns the accession's data in the
    form of a dictionary

    upid: A valid Uniprot's proteome id
    assembly_acc: A valid ENA's WGS accession
    kingdom: A string representing the kingdom a species belongs to (e.g.viruses)
    """

    wgs_entry = {}
    fields = {}

    response = requests.get(gc.ENA_XML_URL % assembly_acc).content
    assembly_xml = ET.fromstring(response)  # root
    entry = assembly_xml.find("entry")

    fields["gca_acc"] = None
    fields["gca_version"] = None

    fields["wgs_acc"] = entry.get("accession")
    fields["wgs_version"] = int(entry.get("version"))
    fields["ensembl_id"] = None  # post processing

    # fetching assembly name out of entry's comment lines

    entry_cmnt = None
    entry_cmnt = entry.find("comment")

    if entry_cmnt is not None:
        assembly_name = get_wgs_assembly_name(
            entry.find("comment").text)
        fields["assembly_name"] = assembly_name

    else:
        fields["assembly_name"] = None

    if entry.get("topology") == "linear":
        fields["circular"] = 0
    else:
        fields["circular"] = 1

    fields["assembly_level"] = "contig"

    fields["study_ref"] = entry.find("projectAccession").text
    fields["description"] = entry.find("reference").find("title").text
    fields["kingdom"] = kingdom

    taxon = entry.find("feature").find("taxon")
    fields["scientific_name"] = taxon.get("scientificName")
    fields["common_name"] = None
    fields["ncbi_id"] = int(taxon.get("taxId"))

    fields["total_length"] = None
    fields["ungapped_length"] = None

    fields["num_gen_regions"] = None

    fields["num_rfam_regions"] = None  # post process
    fields["num_families"] = None  # post process

    # this takes the date of the entry is created
    entry_date = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
    fields["created"] = entry_date
    fields["updated"] = entry_date

    wgs_entry["model"] = gc.GENOME_MODEL
    wgs_entry["pk"] = upid
    wgs_entry["fields"] = fields

    return wgs_entry

# -----------------------------------------------------------------------------

def fetch_wgs_accs_metadata(upid, assembly_acc, wgs_range):
    """
    This function runs over the wgs range and exports all metadata for every
    accession in the provided wgs range

    wgs_range: A valid ENA-WGS set range
    """

    wgs_entries = []
    entry = {}
    fields = {}
    wgs_accs = gf.fetch_wgs_range_accs(wgs_range)

    for acc in wgs_accs:
        entry["model"] = gc.GENSEQ_MODEL
        entry["pk"] = acc
        fields = fetch_wgs_acc_metadata(acc)
        entry["fields"] = fields
        # adding upid and wgs_acc in entry fields
        entry["fields"]["upid"] = upid
        entry["fields"]["gen_acc"] = assembly_acc
        wgs_entries.append(entry)
        entry = {}

    return wgs_entries

# -----------------------------------------------------------------------------

def fetch_wgs_acc_metadata(wgs_acc):
    """
    Return a fields dictionary

    wgs_acc: A valid ENA wgs accession
    """

    fields = {}
    response = requests.get(gf.ENA_XML_URL % wgs_acc).content
    acc_xml = ET.fromstring(response)
    entry = acc_xml.find("entry")

    fields["seq_version"] = int(entry.get("entryVersion"))
    fields["seq_length"] = int(entry.get("sequenceLength"))
    fields["seq_role"] = "contig"
    fields["mol_type"] = entry.get("moleculeType")

    taxon_node = None
    taxon_node = entry.find("feature").find("taxon")
    fields["ncbi_id"] = int(taxon_node.get("taxId"))

    location = None
    refs = None
    refs = entry.findall("reference")

    for ref in refs:
        location = ref.get("location")
        if location is not None:
            break

    if location is not None:
        location = location.strip().split('-')
        fields["seq_start"] = int(location[0])
        fields["seq_end"] = int(location[1])
    else:
        fields["seq_start"] = 0
        fields["seq_end"] = 0

    fields["assembly_unit"] = None  # need to check this one
    fields["description"] = entry.find("description").text

    # this takes the date of the entry is created
    entry_date = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
    fields["created"] = entry_date
    fields["updated"] = entry_date  # when adding a new genome

    return fields
# -----------------------------------------------------------------------------

def get_wgs_assembly_name(wgs_comment):
    """
    Search the wgs comment text and return the Assembly name

    wgs_comment: WGS comment text
    """

    assembly_name = ''
    cmnt_lines = wgs_comment.strip().split('\n')

    for line in cmnt_lines:
        if line.find("Assembly Name") != -1:
            assembly_name = line
            break

    assembly_name = assembly_name.strip().partition("::")[2].strip()

    return assembly_name
# -----------------------------------------------------------------------------

def fetch_assembly_attributes(attrs_node):
    """
    Runs over the attributes node in the xml file and returns all attribute
    tag-value pairs in a dictionary format

    attrs_node: GCA xml's attributes node
    """

    attribute_values = {}
    attributes = attrs_node.findall("ASSEMBLY_ATTRIBUTE")

    for attr in attributes:
        tag = attr.find("TAG").text
        value = attr.find("VALUE").text
        attribute_values[tag] = value

    return attribute_values

# -----------------------------------------------------------------------------

def fetch_gca_acc_metadata(accession):
    """
    Fetch accession metadata and return a dictionary with ncbi_id, molecule's
    type, description

    accession: A valid GCA related ENA accession
    """

    metadata = {}
    xml_str = requests.get(gc.ENA_XML_URL % accession).content

    xml_root = ET.fromstring(xml_str)

    entry = xml_root.find("entry")

    '''
    mol_type = None
    mol_type = entry.get("moleculeType")
    '''

    # None if no moleculeType found
    metadata["mol_type"] = entry.get("moleculeType")
    metadata["description"] = entry.find("description").text
    metadata["ncbi_id"] = int(entry.find("feature").find("taxon").get("taxId"))

    return metadata

# -----------------------------------------------------------------------------

def fetch_assembly_links(gca_acc):
    """
    Retrieves and returns a dictionary with all ftp links found in the GCA xml
    file.

    gca_acc: A valid ENA GCA accession
    """

    gca_ftp_links = {}

    response = requests.get(gc.ENA_XML_URL % gca_acc).content

    xml = ET.fromstring(response)

    # fetch assembly links node
    assembly_links = xml.find("ASSEMBLY").find(
        "ASSEMBLY_LINKS").findall("ASSEMBLY_LINK")

    # loop over all available links
    for link in assembly_links:
        label = link.find("URL_LINK").find("LABEL").text.replace(' ', '_')
        url = link.find("URL_LINK").find("URL").text

        gca_ftp_links[label] = url

        label = None
        url = None

    return gca_ftp_links

# -----------------------------------------------------------------------------
if __name__ == '__main__':
    pass
<|endoftext|>


<commit_before>#This file is part of QuTiP.
#
#    QuTIP is free software: you can redistribute it and/or modify
#    it under the terms of the GNU General Public License as published by
#    the Free Software Foundation, either version 3 of the License, or
#   (at your option) any later version.
#
#    QuTIP is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU General Public License for more details.
#
#    You should have received a copy of the GNU General Public License
#    along with QuTIP.  If not, see <http://www.gnu.org/licenses/>.
#
# Copyright (C) 2011, Paul D. Nation & Robert J. Johansson
#
###########################################################################
#
# Textbook example: Heisenberg spin 1/2 chain
#
#
from scipy import *
from ..Qobj import *
from ..states import *
from ..operators import *
from ..ode_solve import *
from ..mcsolve import *
from ..tensor import *
from ..ptrace import *
from pylab import *
from termpause import termpause
import time
def spinchain():
    print 'Calculates the sigma-z expectation values for a'
    print 'Heisenberg spin-chain with four qubits.'
    termpause()
    def integrate(N, h, Jx, Jy, Jz, psi0, tlist, gamma, solver):

        # Hamiltonian
        #
        # H = - 0.5 sum_n^N h_n sigma_z(n)
        #     - 0.5 sum_n^(N-1) [ Jx_n sigma_x(n) sigma_x(n+1) + Jy_n sigma_y(n) sigma_y(n+1) + Jz_n sigma_z(n) sigma_z(n+1)]
        #
        si = qeye(2)
        sx = sigmax()
        sy = sigmay()
        sz = sigmaz()
        sx_list = []
        sy_list = []
        sz_list = []

        for n in xrange(N):
            op_list = []
            for m in xrange(N):
                op_list.append(si)
            op_list[n] = sx
            sx_list.append(tensor(op_list))
            op_list[n] = sy
            sy_list.append(tensor(op_list))
            op_list[n] = sz
            sz_list.append(tensor(op_list))

        H = 0
        # energy splitting terms
        for n in xrange(N):
            H += - 0.5 * h[n] * sz_list[n]
        # interaction terms
        for n in xrange(N-1):
            H += - 0.5 * Jx[n] * sx_list[n] * sx_list[n+1]
            H += - 0.5 * Jy[n] * sy_list[n] * sy_list[n+1]
            H += - 0.5 * Jz[n] * sz_list[n] * sz_list[n+1]

        # collapse operators
        c_op_list = []
        # spin dephasing
        for n in xrange(N):
            if gamma[n] > 0.0:
                c_op_list.append(sqrt(gamma[n]) * sz_list[n])
        # evolve and calculate expectation values
        if solver == "ode":
            expt_list = me_ode_solve(H, psi0, tlist, c_op_list, sz_list)
        elif solver == "mc":
            ntraj = 250
            expt_list = mcsolve(H, psi0, tlist, ntraj, c_op_list, sz_list)
        return expt_list

    #--------------------------------------
    # set up the calculation
    print "set up the calculation"
    print "solver = 'ode'   # use the ode solver"
    print "N = 4 # number of spins"
    print "# array of spin energy splittings and coupling strengths. here we use"
    print "# uniform parameters, but in general we don't have too"
    print "h  = 1.0 * 2 * pi * ones(N) "
    print "Jz = 0.1 * 2 * pi * ones(N)"
    print "Jx = 0.1 * 2 * pi * ones(N)"
    print "Jy = 0.1 * 2 * pi * ones(N)"
    print "# dephasing rate"
    print "gamma = 0.01 * ones(N)"
    print "# intial state, first spin in state |1>, the rest in state |0>"
    print "psi_list = []"
    print "psi_list.append(basis(2,1))"
    print "for n in xrange(N-1):"
    print "    psi_list.append(basis(2,0))"
    print "psi0 = tensor(psi_list)"
    print "tlist = linspace(0, 50, 300)"
    print "start_time = time.time()"
    print "sz_expt = integrate(N, h, Jx, Jy, Jz, psi0, tlist, gamma, solver)"
    print "print 'time elapsed = ' +str(time.time() - start_time)"

    solver = "ode"   # use the ode solver
    #solver = "mc"   # use the monte-carlo solver
    N = 4 # number of spins
    # array of spin energy splittings and coupling strengths. here we use
    # uniform parameters, but in general we don't have too
    h  = 1.0 * 2 * pi * ones(N)
    Jz = 0.1 * 2 * pi * ones(N)
    Jx = 0.1 * 2 * pi * ones(N)
    Jy = 0.1 * 2 * pi * ones(N)
    # dephasing rate
    gamma = 0.01 * ones(N)
    # intial state, first spin in state |1>, the rest in state |0>
    psi_list = []
    psi_list.append(basis(2,1))
    for n in xrange(N-1):
        psi_list.append(basis(2,0))
    psi0 = tensor(psi_list)
    tlist = linspace(0, 50, 300)
    start_time = time.time()
    sz_expt = integrate(N, h, Jx, Jy, Jz, psi0, tlist, gamma, solver)
    print 'time elapsed = ' +str(time.time() - start_time)

    print '\nPlot the results....'
    termpause()
    print "rc('font', family='Bitstream Vera Sans')"
    print "for n in xrange(N):"
    print "  plot(tlist, real(sz_expt[n,:]), label=r'$\langle\sigma_z($'+str(n)+r'$)\rangle$')"
    print "xlabel(r'Time [ns]',fontsize=14)"
    print "ylabel(r'$\langle\sigma_{z}\rangle$',fontsize=14)"
    print "title(r'Dynamics of a Heisenberg spin chain')"
    print "legend(loc = 'lower right')"
    print "show()"

    rc('font', family='Bitstream Vera Sans')
    for n in xrange(N):
        plot(tlist, real(sz_expt[n,:]), label=r'$\langle\sigma_z($'+str(n)+r'$)\rangle$',lw=2)
    xlabel(r'Time [ns]',fontsize=14)
    ylabel(r'$\langle\sigma_{z}\rangle$',fontsize=14)
    title(r'Dynamics of a Heisenberg spin chain')
    legend(loc = "lower right")
    show()

if __name__=="main()":
    spinchain()

<commit_msg>small fix in indexing<commit_after>#This file is part of QuTiP.
#
#    QuTIP is free software: you can redistribute it and/or modify
#    it under the terms of the GNU General Public License as published by
#    the Free Software Foundation, either version 3 of the License, or
#   (at your option) any later version.
#
#    QuTIP is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU General Public License for more details.
#
#    You should have received a copy of the GNU General Public License
#    along with QuTIP.  If not, see <http://www.gnu.org/licenses/>.
#
# Copyright (C) 2011, Paul D. Nation & Robert J. Johansson
#
###########################################################################
#
# Textbook example: Heisenberg spin 1/2 chain
#
#
from scipy import *
from ..Qobj import *
from ..states import *
from ..operators import *
from ..ode_solve import *
from ..mcsolve import *
from ..tensor import *
from ..ptrace import *
from pylab import *
from termpause import termpause
import time
def spinchain():
    print 'Calculates the sigma-z expectation values for a'
    print 'Heisenberg spin-chain with four qubits.'
    termpause()
    def integrate(N, h, Jx, Jy, Jz, psi0, tlist, gamma, solver):

        # Hamiltonian
        #
        # H = - 0.5 sum_n^N h_n sigma_z(n)
        #     - 0.5 sum_n^(N-1) [ Jx_n sigma_x(n) sigma_x(n+1) + Jy_n sigma_y(n) sigma_y(n+1) + Jz_n sigma_z(n) sigma_z(n+1)]
        #
        si = qeye(2)
        sx = sigmax()
        sy = sigmay()
        sz = sigmaz()
        sx_list = []
        sy_list = []
        sz_list = []

        for n in xrange(N):
            op_list = []
            for m in xrange(N):
                op_list.append(si)
            op_list[n] = sx
            sx_list.append(tensor(op_list))
            op_list[n] = sy
            sy_list.append(tensor(op_list))
            op_list[n] = sz
            sz_list.append(tensor(op_list))

        H = 0
        # energy splitting terms
        for n in xrange(N):
            H += - 0.5 * h[n] * sz_list[n]
        # interaction terms
        for n in xrange(N-1):
            H += - 0.5 * Jx[n] * sx_list[n] * sx_list[n+1]
            H += - 0.5 * Jy[n] * sy_list[n] * sy_list[n+1]
            H += - 0.5 * Jz[n] * sz_list[n] * sz_list[n+1]

        # collapse operators
        c_op_list = []
        # spin dephasing
        for n in xrange(N):
            if gamma[n] > 0.0:
                c_op_list.append(sqrt(gamma[n]) * sz_list[n])
        # evolve and calculate expectation values
        if solver == "ode":
            expt_list = me_ode_solve(H, psi0, tlist, c_op_list, sz_list)
        elif solver == "mc":
            ntraj = 250
            expt_list = mcsolve(H, psi0, tlist, ntraj, c_op_list, sz_list)
        return expt_list

    #--------------------------------------
    # set up the calculation
    print "set up the calculation"
    print "solver = 'ode'   # use the ode solver"
    print "N = 4 # number of spins"
    print "# array of spin energy splittings and coupling strengths. here we use"
    print "# uniform parameters, but in general we don't have too"
    print "h  = 1.0 * 2 * pi * ones(N) "
    print "Jz = 0.1 * 2 * pi * ones(N)"
    print "Jx = 0.1 * 2 * pi * ones(N)"
    print "Jy = 0.1 * 2 * pi * ones(N)"
    print "# dephasing rate"
    print "gamma = 0.01 * ones(N)"
    print "# intial state, first spin in state |1>, the rest in state |0>"
    print "psi_list = []"
    print "psi_list.append(basis(2,1))"
    print "for n in xrange(N-1):"
    print "    psi_list.append(basis(2,0))"
    print "psi0 = tensor(psi_list)"
    print "tlist = linspace(0, 50, 300)"
    print "start_time = time.time()"
    print "sz_expt = integrate(N, h, Jx, Jy, Jz, psi0, tlist, gamma, solver)"
    print "print 'time elapsed = ' +str(time.time() - start_time)"

    solver = "ode"   # use the ode solver
    #solver = "mc"   # use the monte-carlo solver
    N = 4 # number of spins
    # array of spin energy splittings and coupling strengths. here we use
    # uniform parameters, but in general we don't have too
    h  = 1.0 * 2 * pi * ones(N)
    Jz = 0.1 * 2 * pi * ones(N)
    Jx = 0.1 * 2 * pi * ones(N)
    Jy = 0.1 * 2 * pi * ones(N)
    # dephasing rate
    gamma = 0.01 * ones(N)
    # intial state, first spin in state |1>, the rest in state |0>
    psi_list = []
    psi_list.append(basis(2,1))
    for n in xrange(N-1):
        psi_list.append(basis(2,0))
    psi0 = tensor(psi_list)
    tlist = linspace(0, 50, 300)
    start_time = time.time()
    sz_expt = integrate(N, h, Jx, Jy, Jz, psi0, tlist, gamma, solver)
    print 'time elapsed = ' +str(time.time() - start_time)

    print '\nPlot the results....'
    termpause()
    print "rc('font', family='Bitstream Vera Sans')"
    print "for n in xrange(N):"
    print "  plot(tlist, real(sz_expt[n,:]), label=r'$\langle\sigma_z($'+str(n)+r'$)\rangle$')"
    print "xlabel(r'Time [ns]',fontsize=14)"
    print "ylabel(r'$\langle\sigma_{z}\rangle$',fontsize=14)"
    print "title(r'Dynamics of a Heisenberg spin chain')"
    print "legend(loc = 'lower right')"
    print "show()"

    rc('font', family='Bitstream Vera Sans')
    for n in xrange(N):
        plot(tlist, real(sz_expt[n]), label=r'$\langle\sigma_z($'+str(n)+r'$)\rangle$',lw=2)
    xlabel(r'Time [ns]',fontsize=14)
    ylabel(r'$\langle\sigma_{z}\rangle$',fontsize=14)
    title(r'Dynamics of a Heisenberg spin chain')
    legend(loc = "lower right")
    show()

if __name__=="main()":
    spinchain()

<|endoftext|>


<commit_before>#!/usr/bin/env python

# Displays all Launchpad bugs for OpenStack/Nova which block
# the creation of the release candidate.
#
# Copyright 2016 Markus Zoeller

import argparse
import os
import datetime
import common

parser = argparse.ArgumentParser()
parser.add_argument('-p',
                    '--project-name',
                    required=True,
                    dest='project_name',
                    help='The LP project name.')

args = parser.parse_args()

PROJECT_NAME = args.project_name

RECENT_ACTIVITY_IN_DAYS = 14

start = datetime.datetime.now()

project = common.get_project_client(PROJECT_NAME)

ALL_STATES = ["New", "Incomplete", "Confirmed", "Won't Fix", "Opinion",
              "Invalid", "Fix Released"]
bug_tasks = project.searchTasks(status=ALL_STATES,
                                order_by='-date_last_updated',
                                omit_duplicates=True)

# bug_task
# https://api.launchpad.net/1.0/bugs/1533876

# bug_task.bug
# https://api.launchpad.net/1.0/nova/+bug/1533876

# bug_task.bug.related_tasks
# https://api.launchpad.net/1.0/nova/+bug/1533876/related_tasks

class StatSummary(object):

    def __init__(self, person_url, person_name):
        self.person_url = person_url.encode('ascii', 'replace')
        self.person_name = person_name.encode('ascii', 'replace')
        self.created_reports = []
        self.confirmed_reports = []
        self.rejected_reports = []
        self.resolved_reports = []
        self.inquired_reports = []

    def __eq__(self, other):
        return (isinstance(other, self.__class__) and
               self.person_url == other.person_url)

    def __cmp__(self,other):
        return cmp(self.person_name, other.person_name)

    def __repr__(self):
        return "<StatSummary: person=%s, created=%d, confirmed=%d, rejected=%d, resolved=%d, inquired=%d, sum=%d>" % \
            (self.person_name, self.created, self.confirmed, self.rejected, self.resolved, self.inquired, self.sum)

    @property
    def created(self):
        return len(self.created_reports)

    @property
    def confirmed(self):
        return len(self.confirmed_reports)

    @property
    def rejected(self):
        return len(self.rejected_reports)

    @property
    def resolved(self):
        return len(self.resolved_reports)

    @property
    def inquired(self):
        return len(self.inquired_reports)

    @property
    def sum(self):
        return self.created + self.confirmed + self.rejected + \
            self.resolved + self.inquired

print "================================================="
print " recent bug actions"
print "================================================="
today = datetime.datetime.today()
counter = 0
stats = {}

def get_summary(person):
    link = "unknown"
    name = "unknown"
    if person:
        link = person.web_link
        name = person.display_name
    if not link in stats.keys():
        stats[link] = StatSummary(link, name)
    return stats[link]

def is_created(bug_task, a):
    return (a.whatchanged =="bug" and a.message == "added bug" and \
            bug_task.date_created == bug_task.bug.date_created) or \
            a.whatchanged == "bug task added" and a.newvalue == "nova"

def is_rejected(a):
    return a.newvalue in ["Invalid", "Opinion", "Won't Fix"]

def is_new_confirmed(a):
    return a.newvalue == 'Confirmed' and a.oldvalue == "New"

def is_trackable_status_change(a):
    return a.whatchanged == "nova: status" and \
           a.newvalue in ["Incomplete", "Confirmed", "Won't Fix", "Opinion",
                          "Invalid", "Fix Released"]

for bug_task in bug_tasks:
    diff = today - bug_task.bug.date_last_updated.replace(tzinfo=None)
    if diff.days > RECENT_ACTIVITY_IN_DAYS:
        break

    print("%d - %d days" % (counter, diff.days))
    for a in bug_task.bug.activity:
        diff = today - a.datechanged.replace(tzinfo=None)
        if diff.days > RECENT_ACTIVITY_IN_DAYS:
            # ignore activities which are not recent
            continue

        person = a.person

        if is_created(bug_task, a):
            get_summary(person).created_reports.append(bug_task.web_link)
        elif is_trackable_status_change(a):
            if is_rejected(a):
                get_summary(person).rejected_reports.append(bug_task.web_link)
            elif is_new_confirmed(a):
                get_summary(person).confirmed_reports.append(bug_task.web_link)
            elif a.newvalue == 'Fix Released':
                if person.web_link.encode('ascii', 'replace') == "https://launchpad.net/~hudson-openstack":
                    person = bug_task.assignee if bug_task.assignee else person
                get_summary(person).resolved_reports.append(bug_task.web_link)
            elif a.newvalue == "Incomplete":
                get_summary(person).inquired_reports.append(bug_task.web_link)
            #
    counter += 1

print(counter)

for s in sorted(stats.values()):
    print(str(s))

end = datetime.datetime.now()
print("Duration: %s" % (end - start).seconds)
<commit_msg>query_recent_bug_triage: Do not hard-code "nova" name<commit_after>#!/usr/bin/env python

# Displays all Launchpad bugs for OpenStack/Nova which block
# the creation of the release candidate.
#
# Copyright 2016 Markus Zoeller

import argparse
import os
import datetime
import common

parser = argparse.ArgumentParser()
parser.add_argument('-p',
                    '--project-name',
                    required=True,
                    dest='project_name',
                    help='The LP project name.')

args = parser.parse_args()

PROJECT_NAME = args.project_name

RECENT_ACTIVITY_IN_DAYS = 14

start = datetime.datetime.now()

project = common.get_project_client(PROJECT_NAME)

ALL_STATES = ["New", "Incomplete", "Confirmed", "Won't Fix", "Opinion",
              "Invalid", "Fix Released"]
bug_tasks = project.searchTasks(status=ALL_STATES,
                                order_by='-date_last_updated',
                                omit_duplicates=True)

# bug_task
# https://api.launchpad.net/1.0/bugs/1533876

# bug_task.bug
# https://api.launchpad.net/1.0/nova/+bug/1533876

# bug_task.bug.related_tasks
# https://api.launchpad.net/1.0/nova/+bug/1533876/related_tasks

class StatSummary(object):

    def __init__(self, person_url, person_name):
        self.person_url = person_url.encode('ascii', 'replace')
        self.person_name = person_name.encode('ascii', 'replace')
        self.created_reports = []
        self.confirmed_reports = []
        self.rejected_reports = []
        self.resolved_reports = []
        self.inquired_reports = []

    def __eq__(self, other):
        return (isinstance(other, self.__class__) and
               self.person_url == other.person_url)

    def __cmp__(self,other):
        return cmp(self.person_name, other.person_name)

    def __repr__(self):
        return "<StatSummary: person=%s, created=%d, confirmed=%d, rejected=%d, resolved=%d, inquired=%d, sum=%d>" % \
            (self.person_name, self.created, self.confirmed, self.rejected, self.resolved, self.inquired, self.sum)

    @property
    def created(self):
        return len(self.created_reports)

    @property
    def confirmed(self):
        return len(self.confirmed_reports)

    @property
    def rejected(self):
        return len(self.rejected_reports)

    @property
    def resolved(self):
        return len(self.resolved_reports)

    @property
    def inquired(self):
        return len(self.inquired_reports)

    @property
    def sum(self):
        return self.created + self.confirmed + self.rejected + \
            self.resolved + self.inquired

print "================================================="
print " recent bug actions"
print "================================================="
today = datetime.datetime.today()
counter = 0
stats = {}

def get_summary(person):
    link = "unknown"
    name = "unknown"
    if person:
        link = person.web_link
        name = person.display_name
    if not link in stats.keys():
        stats[link] = StatSummary(link, name)
    return stats[link]

def is_created(bug_task, a):
    return (a.whatchanged =="bug" and a.message == "added bug" and \
            bug_task.date_created == bug_task.bug.date_created) or \
            a.whatchanged == "bug task added" and a.newvalue == PROJECT_NAME

def is_rejected(a):
    return a.newvalue in ["Invalid", "Opinion", "Won't Fix"]

def is_new_confirmed(a):
    return a.newvalue == 'Confirmed' and a.oldvalue == "New"

def is_trackable_status_change(a):
    return a.whatchanged == PROJECT_NAME + ": status" and \
           a.newvalue in ["Incomplete", "Confirmed", "Won't Fix", "Opinion",
                          "Invalid", "Fix Released"]

for bug_task in bug_tasks:
    diff = today - bug_task.bug.date_last_updated.replace(tzinfo=None)
    if diff.days > RECENT_ACTIVITY_IN_DAYS:
        break

    print("%d - %d days" % (counter, diff.days))
    for a in bug_task.bug.activity:
        diff = today - a.datechanged.replace(tzinfo=None)
        if diff.days > RECENT_ACTIVITY_IN_DAYS:
            # ignore activities which are not recent
            continue

        person = a.person

        if is_created(bug_task, a):
            get_summary(person).created_reports.append(bug_task.web_link)
        elif is_trackable_status_change(a):
            if is_rejected(a):
                get_summary(person).rejected_reports.append(bug_task.web_link)
            elif is_new_confirmed(a):
                get_summary(person).confirmed_reports.append(bug_task.web_link)
            elif a.newvalue == 'Fix Released':
                if person.web_link.encode('ascii', 'replace') == "https://launchpad.net/~hudson-openstack":
                    person = bug_task.assignee if bug_task.assignee else person
                get_summary(person).resolved_reports.append(bug_task.web_link)
            elif a.newvalue == "Incomplete":
                get_summary(person).inquired_reports.append(bug_task.web_link)
            #
    counter += 1

print(counter)

for s in sorted(stats.values()):
    print(str(s))

end = datetime.datetime.now()
print("Duration: %s" % (end - start).seconds)
<|endoftext|>


<commit_before>from datetime import datetime

from django.db import models
from django.dispatch import dispatcher
from django.db.models import signals
from django.utils.html import escape
from django.utils.translation import ugettext_lazy as _

from django.contrib.auth.models import User

try:
    from notification import models as notification
except ImportError:
    notification = None

# @@@ this is based on Tribes -- can we re-use anything?

class Project(models.Model):
    """
    a project is a group of users working together on a common set of tasks.
    """

    slug = models.SlugField(_('slug'), unique=True)
    name = models.CharField(_('name'), max_length=80, unique=True)
    creator = models.ForeignKey(User, related_name="created_projects", verbose_name=_('creator'))
    created = models.DateTimeField(_('created'), default=datetime.now)
    description = models.TextField(_('description'))
    members = models.ManyToManyField(User, verbose_name=_('members'))

    # private means only members can see the project
    private =models.BooleanField(_('private'), default=False)

    def __unicode__(self):
        return self.name

    @models.permalink
    def get_absolute_url(self):
        return ("tribe_detail", [self.slug])

    class Admin:
        list_display = ('name', 'creator', 'created', 'private')

class Topic(models.Model):
    """
    a discussion topic for the project.
    """

    project = models.ForeignKey(Project, related_name="topics", verbose_name=_('project'))

    title = models.CharField(_('title'), max_length="50")
    creator = models.ForeignKey(User, related_name="created_project_topics", verbose_name=_('creator'))
    created = models.DateTimeField(_('created'), default=datetime.now)
    modified = models.DateTimeField(_('modified'), default=datetime.now) # topic modified when commented on
    body = models.TextField(_('body'), blank=True)

    def __unicode__(self):
        return self.title

    @models.permalink
    def get_absolute_url(self):
        return ("project_topic", [self.pk])

    class Meta:
        ordering = ('-modified', )

    class Admin:
        list_display = ('title', )

class Task(models.Model):
    """
    a task to be performed for the project.
    """

    STATE_CHOICES = (
        ('1', 'open'),
        ('2', 'resolved'), # the assignee thinks it's done
        ('3', 'closed'), # the creator has confirmed it's done
    )

    project = models.ForeignKey(Project, related_name="tasks", verbose_name=_('project'))

    summary = models.CharField(_('summary'), max_length="100")
    detail = models.TextField(_('detail'), blank=True)
    creator = models.ForeignKey(User, related_name="created_project_tasks", verbose_name=_('creator'))
    created = models.DateTimeField(_('created'), default=datetime.now)
    modified = models.DateTimeField(_('modified'), default=datetime.now) # task modified when commented on or when various fields changed
    assignee = models.ForeignKey(User, related_name="assigned_project_tasks", verbose_name=_('assignee'), null=True, blank=True)

    # status is a short message the assignee can give on their current status
    status = models.CharField(_('status'), max_length="50", blank=True)
    state = models.CharField(_('state'), max_length="1", choices=STATE_CHOICES, default=1)

    def __unicode__(self):
        return self.summary

    def save(self):
        self.modified = datetime.now()
        super(Task, self).save()

    @models.permalink
    def get_absolute_url(self):
        return ("project_task", [self.pk])

    class Admin:
        pass

from threadedcomments.models import ThreadedComment
def new_comment(sender, instance):
    if isinstance(instance.content_object, Topic):
        topic = instance.content_object
        topic.modified = datetime.now()
        topic.save()
        if notification:
            notification.send([topic.creator], "projects_topic_response", {"user": instance.user, "topic": topic})
    elif isinstance(instance.content_object, Task):
        task = instance.content_object
        task.modified = datetime.now()
        task.save()
        project = task.project
        if notification:
            notification.send(project.members.all(), "projects_task_comment", {"user": instance.user, "task": task, "project": project})
dispatcher.connect(new_comment, signal=signals.post_save, sender=ThreadedComment)
<commit_msg>fixed get_absolute_url of project<commit_after>from datetime import datetime

from django.db import models
from django.dispatch import dispatcher
from django.db.models import signals
from django.utils.html import escape
from django.utils.translation import ugettext_lazy as _

from django.contrib.auth.models import User

try:
    from notification import models as notification
except ImportError:
    notification = None

# @@@ this is based on Tribes -- can we re-use anything?

class Project(models.Model):
    """
    a project is a group of users working together on a common set of tasks.
    """

    slug = models.SlugField(_('slug'), unique=True)
    name = models.CharField(_('name'), max_length=80, unique=True)
    creator = models.ForeignKey(User, related_name="created_projects", verbose_name=_('creator'))
    created = models.DateTimeField(_('created'), default=datetime.now)
    description = models.TextField(_('description'))
    members = models.ManyToManyField(User, verbose_name=_('members'))

    # private means only members can see the project
    private = models.BooleanField(_('private'), default=False)

    def __unicode__(self):
        return self.name

    @models.permalink
    def get_absolute_url(self):
        return ("project_detail", [self.slug])

    class Admin:
        list_display = ('name', 'creator', 'created', 'private')

class Topic(models.Model):
    """
    a discussion topic for the project.
    """

    project = models.ForeignKey(Project, related_name="topics", verbose_name=_('project'))

    title = models.CharField(_('title'), max_length="50")
    creator = models.ForeignKey(User, related_name="created_project_topics", verbose_name=_('creator'))
    created = models.DateTimeField(_('created'), default=datetime.now)
    modified = models.DateTimeField(_('modified'), default=datetime.now) # topic modified when commented on
    body = models.TextField(_('body'), blank=True)

    def __unicode__(self):
        return self.title

    @models.permalink
    def get_absolute_url(self):
        return ("project_topic", [self.pk])

    class Meta:
        ordering = ('-modified', )

    class Admin:
        list_display = ('title', )

class Task(models.Model):
    """
    a task to be performed for the project.
    """

    STATE_CHOICES = (
        ('1', 'open'),
        ('2', 'resolved'), # the assignee thinks it's done
        ('3', 'closed'), # the creator has confirmed it's done
    )

    project = models.ForeignKey(Project, related_name="tasks", verbose_name=_('project'))

    summary = models.CharField(_('summary'), max_length="100")
    detail = models.TextField(_('detail'), blank=True)
    creator = models.ForeignKey(User, related_name="created_project_tasks", verbose_name=_('creator'))
    created = models.DateTimeField(_('created'), default=datetime.now)
    modified = models.DateTimeField(_('modified'), default=datetime.now) # task modified when commented on or when various fields changed
    assignee = models.ForeignKey(User, related_name="assigned_project_tasks", verbose_name=_('assignee'), null=True, blank=True)

    # status is a short message the assignee can give on their current status
    status = models.CharField(_('status'), max_length="50", blank=True)
    state = models.CharField(_('state'), max_length="1", choices=STATE_CHOICES, default=1)

    def __unicode__(self):
        return self.summary

    def save(self):
        self.modified = datetime.now()
        super(Task, self).save()

    @models.permalink
    def get_absolute_url(self):
        return ("project_task", [self.pk])

    class Admin:
        pass

from threadedcomments.models import ThreadedComment
def new_comment(sender, instance):
    if isinstance(instance.content_object, Topic):
        topic = instance.content_object
        topic.modified = datetime.now()
        topic.save()
        if notification:
            notification.send([topic.creator], "projects_topic_response", {"user": instance.user, "topic": topic})
    elif isinstance(instance.content_object, Task):
        task = instance.content_object
        task.modified = datetime.now()
        task.save()
        project = task.project
        if notification:
            notification.send(project.members.all(), "projects_task_comment", {"user": instance.user, "task": task, "project": project})
dispatcher.connect(new_comment, signal=signals.post_save, sender=ThreadedComment)
<|endoftext|>