<commit_before>__author__ = 'Chris Krycho'
__copyright__ = '2013 Chris Krycho'

from logging import error
from os import path, walk
from sys import exit

try:
    from markdown import Markdown
    from mixins import DictAsMember

except ImportError as import_error:
    error(import_error)
    exit()

def convert_source(config):
    '''
    Convert all Markdown pages to HTML and metadata pairs. Pairs are keyed to
    file names slugs (without the original file extension).
    '''

    md = Markdown(extensions=config.markdown_extensions, output_format='html5')
    converted = {}
    for root, dirs, file_names in walk(config.site.content.source):
        for file_name in file_names:
            file_path = path.join(root, file_name)
            md_text = open(file_path, 'r').read()
            content = md.convert(md_text)

            plain_slug, extension = path.splitext(file_name)
            converted[plain_slug] = {'content': content, 'meta': md.Meta}

    return DictAsMember(converted)
<commit_msg>Fix file open/read/close - there was no close() call in previous version! Use context handler instead.<commit_after>__author__ = 'Chris Krycho'
__copyright__ = '2013 Chris Krycho'

from logging import error
from os import path, walk
from sys import exit

try:
    from markdown import Markdown
    from mixins import DictAsMember

except ImportError as import_error:
    error(import_error)
    exit()

def convert_source(config):
    '''
    Convert all Markdown pages to HTML and metadata pairs. Pairs are keyed to
    file names slugs (without the original file extension).
    '''

    md = Markdown(extensions=config.markdown_extensions, output_format='html5')
    converted = {}
    for root, dirs, file_names in walk(config.site.content.source):
        for file_name in file_names:
            file_path = path.join(root, file_name)
            plain_slug, extension = path.splitext(file_name)

            with open(file_path) as file:
                md_text = file.read()
                content = md.convert(md_text)
                converted[plain_slug] = {'content': content, 'meta': md.Meta}

    return DictAsMember(converted)
<|endoftext|>


<commit_before><commit_msg>params: real width and height screen evaluators added<commit_after><|endoftext|>


<commit_before>"""
Code for python planner
"""

import blockSim as BlockDet
import twoWayDict as twd
import navigation.nav as nav
from datetime import datetime
import comm.serial_interface as comm

class Planner:
  nextSeaLandBlock = [] #list of the next available sea or land block to pick up
  nextAirBlock = [] #list of the 2 air blocks in storage
  armList = [] #list of which arm contains what
  armID = [2, 0]  # armID[0] = right arm, armID[1] = left arm

  storageSim = []
  seaBlockSim = {}
  landBlockSim = {}
  airBlockSim = []

  nextSeaBlockLoc = []
  nextLandBlockLoc = []
  nextAirBlockLoc = []

  scannedSeaLocs = {}
  scannedLandLocs = {}
  colors = []
  #nsbl = twd.TwoWayDict()
  #nlbl = twd.TwoWayDict()

  def __init__(self, bot_loc, blobs, blocks, zones, waypoints, scPlanner, bot_state, qMove_nav):
    """Setup navigation class

    :param bot_loc: Shared dict updated with best-guess location of bot by localizer
    :param blocks: list of blocks
    :param zones:
    :param waypoints: Multiprocessing.Queue object for passing movement feedback to localizer from navigator
    :param si: Serial interface object for sending commands to low-level boards
    :param bot_state: Dict of information about the current state of the bot (ex macro/micro nav)
    :param qMove_nav: Multiprocessing.Queue object for passing movement commands to navigation (mostly from Planner)
    """
    # Store passed-in data
    self.bot_loc = bot_loc
    self.blobs = blobs
    self.blocks = blocks
    self.zones = zones
    self.waypoints = waypoints
    self.scPlanner = scPlanner
    self.bot_state = bot_state
    self.qMove_nav = qMove_nav

    self.armID[0] = comm.right_arm
    self.armID[1] = comm.left_arm

    self.nextSeaLandBlock = ["St01","St02","St03","St04","St05","St06","St07","St08","St09","St10","St11","St12","St13","St14"]
    self.nextAirBlock = []
    self.nextSeaBlockLoc = ["Se01","Se02","Se03","Se04","Se05","Se06"]
    self.nextLandBlockLoc = ["L01","L02","L03","L04","L05","L06"]
    self.colors = ["red", "blue", "green", "orange", "brown", "yellow"]

    for i in range(len(self.nextSeaLandBlock)):
      self.blocks[i] = 1
    for i in range(len(self.nextSeaBlockLoc)):
      self.zones[i] = 0
    for i in range(len(self.nextLandBlockLoc)):
      self.zones[i] = 0
    for i in range(len(self.colors)):
      self.scannedSeaLocs[self.colors[i]] = "empty"
      self.scannedLandLocs[self.colors[i]] = "empty"

  #get current location
  def getCurrentLocation(self):
    return self.bot_loc

  #more to the next block - use this if next block location is
  #handled by nav instead of planner
  def moveToNextBlock(self):
    print "Moving to Next Block"
    self.moveTo(self.getCurrentLocation(), "nextblock loc")
    #micro or macro???

  #move from start to end
  def moveToWayPoint(self, startLoc, endLoc):
    print "Moving from ", startLoc, " to ", endLoc, "--", self.waypoints[endLoc]
    x, y = self.waypoints[endLoc][1]
    theta = self.waypoints[endLoc][2]
    speed = self.waypoints[endLoc][3]
    self.bot_state["naving"] = True
    macro_m = nav.macro_move(x, y, theta, datetime.now())
    self.qMove_nav.put(macro_m)
    while self.bot_state["naving"] != False:
      continue

  def microMove(self, distance):
    print "Moving from ", startLoc, " to ", endLoc
	micro_m = nav.microMoveXY(distance, comm.default_speed)
    self.qMove_nav.put(micro_m)

  def processSeaLand(self):
    armCount = 0
    armList = []
    print "+++++ +++++ +++++ +++++ +++++ +++++"
    for i in range(len(self.nextSeaLandBlock)):
      stID = self.nextSeaLandBlock[i];
      print "Processing: [", stID, self.waypoints[stID], "]"
      self.moveToWayPoint(self.getCurrentLocation(), stID)

      #get block from vision
      block = self.storageSim[i]
      #print "Processing: [", block.getColor(), block.getSize(), block.getLocation(), "]"

      #if the block is small, assign that to list of air blocks
      #continue / move to next block
      if block.getSize() == "small":
        self.nextAirBlock.append(block)
        continue

      #in order to pick up a block, first check if bot is centered
      #that code can exist in pickUpBlock().
      self.pickUpBlock(armCount) #arm 0 or 1.
      armList.append(block)
      armCount = armCount + 1;

      if armCount == 2:
        print "picked up 2 blocks"

		#when dropping blocks off, offset the center of the bot
		# about 0.5 from the center of the dropoff zone

        #Both arms contain sea blocks
        if armList[0].getSize() == "medium" and armList[1].getSize() == "medium":
          self.moveToWayPoint(self.getCurrentLocation(), "sea")

          goToNextDropOff(armList[0], "sea")
          self.placeBlock(0)

          goToNextDropOff(armList[1], "sea")
          self.placeBlock(1)

        #Both arms contain land blocks
        elif armList[0].getSize() == "large" and armList[1].getSize() == "large":
          self.moveToWayPoint(self.getCurrentLocation(), "land")

          goToNextDropOff(armList[0], "land")
          self.placeBlock(0)

          armId = goToNextDropOff(armList[1], "land")
          self.placeBlock(1)

        #One arm contains sea block and other contains land block
        elif armList[0].getSize() == "medium" and armList[1].getSize() == "large":
          self.moveToWayPoint(self.getCurrentLocation(), "sea")
          goToNextDropOff(armList, "sea")
          self.placeBlock(0)

          self.moveToWayPoint(self.getCurrentLocation(), "land")
          goToNextDropOff(armList, "land")
          self.placeBlock(1)

        #One arm contains land block and other contains sea block
        elif armList[0].getSize() == "large" and armList[1].getSize() == "medium":
          # even if the orders are different, first go to sea then land
          self.moveToWayPoint(self.getCurrentLocation(), "sea")
          goToNextDropOff(armList[1], "sea")
          self.placeBlock(1)

          self.moveToWayPoint(self.getCurrentLocation(), "land")
          goToNextDropOff(armList[0], "land")
          self.placeBlock(0)

        armCount = 0
        armList = []
        print "===== ===== ===== ===== ===== ===== ===== ===== ===== ====="
        self.moveToWayPoint(self.getCurrentLocation(), "storage")
      #end if
    #end for
  #end processSeaLand

  def processAir(self):
    for i in range(len(self.nextAirBlock)):
      block = self.nextAirBlock[i];
      print "Processing: [", block.getColor(), block.getSize(), block.getLocation(), "]"
      self.pickUpBlock(block.getLocation(), i);
    #end for

    print "Move to Ramp, up the ramp, to the drop-off"
    self.moveToWayPoint(self.getCurrentLocation(), "grnd2ramp") #normal speed
    self.moveToWayPoint(self.getCurrentLocation(), "lwrPlt") #ramp speed
    #self.moveToWayPint(self.getCurrentLocation(), "uprRamp") #normal speed
    self.moveToWayPoint(self.getCurrentLocation(), "air") #ramp speed

    print "Scan air drop-off"
    print "Drop Air Blocks"

  def getAvailableSeaDropOffs():
    availList = []
    for i in range(len(nextSeaBlockLoc)):
      if zones[nextSeaBlockLoc[i]] == 0:
        availList.append(nextSeaBlockLoc[i])
    return availList

  def getAvailableLandDropOffs():
    availList = []
    for i in range(len(nextLandBlockLoc)):
      if zones[nextLandBlockLoc[i]] == 0:
        availList.append(nextLandBlockLoc[i])
    return availList

  #go to the next sea dropoff zone
  #separate function to handle sea specific movements
  def goToNextSeaDropOff(self, block):
    # if seaDropLocList is empty, go to Se01
    # else, check if color of either block matches
    availList = getAvailableSeaDropOffs()
    blockColor = block.getColor()

    if scannedSeaLocs[blockColor] == "empty":
      #block location unknown
      for i in range(len(availList)):
        self.moveToWayPoint(self.getCurrentLocation(), availList[i])
        #get the color at waypoint
        color = seaBlockSim[availList[i]]
        if color == blockColor:
          #found color
          break
        else:
          scannedSeaLocs[color] = availList[i]

    else:
      self.moveToWayPoint(self.getCurrentLocation(), scannedSeaLocs[blockColor])

  #go to the next land dropoff zone
  #separate function to handle land specific movements
  def goToNextLandDropOff(self, block):
    availList = getAvailableLandDropOffs()
    blockColor = block.getColor()

    if scannedLandLocs[blockColor] == "empty":
      #block location unknown
      for i in range(len(availList)):
        self.moveToWayPoint(self.getCurrentLocation(), availList[i])
        #get the color at waypoint
        color = LandBlockSim[availList[i]]
        if color == blockColor:
          #found color
          break
        else:
          scannedLandLocs[color] = availList[i]

    else:
      self.moveToWayPoint(self.getCurrentLocation(), scannedLandLocs[blockColor])

  # pick up a block given armID
  def pickUpBlock(self, arm):
    armId = self.armID[arm]
    #self.moveTo(self.getCurrentLocation(), blockLoc)
    #print "Picking Up Block at ", blockLoc, "with Arm", armId
    #call vision to make sure we are centered on the block
    #if we are not centered, micromove
    self.scPlanner.gripperOpen(armId)
    self.scPlanner.armDown(armId)
    self.scPlanner.gripperClose(armId)
    self.scPlanner.armUp(armId)

  # place a block given armID
  def placeBlock(self, arm):
    armId = self.armID[arm]
    #self.moveTo(self.getCurrentLocation(), blockLoc)
    #print "Placing block from ", self.armID[arm], "at", blockLoc
    #call vision to make sure we are centered on the block
    #if we are not centered, micromove
    self.scPlanner.armDown(armId)
    self.scPlanner.gripperOpen(armId)
    self.scPlanner.armUp(armId)
    self.scPlanner.gripperClose(armId)

  #main
  def start(self):
    self.storageSimulator("storage") ##use when vision is not available
    self.dropOffSimulator("sea") ##use when vision is not available
    self.dropOffSimulator("land")
    print "Move to Storage Start"
    self.moveToWayPoint(self.getCurrentLocation(), "storage")
    #print "Scan Storage"
    #self.scanStorageFirstTime("storage") # BUG: This should not be hardcoded. Currently fails.
    #print "Move to Storage Start"
    #self.moveToWayPoint(self.getCurrentLocation(), "storage")
    print "***********************************************"
    print "********** Processing - Sea and Land **********"
    print "***********************************************"
    self.processSeaLand()
    print "**************************************"
    print "********** Processing - Air **********"
    print "**************************************"
    self.processAir()

  def test(self):
    print self.waypoints
    print len(self.waypoints)
    print self.waypoints["storage"]

  # Scan the given location for the first time
  # use this if scanning first then dropping blocks
  # def scanFirstTime(self, loc):
#     print "Scanning Storage"
#     print "Initiating Storage Scan"
#     print "updating list of sea, land and air blocks"
#
#     count = 0
#     if loc == "storage":
#       count = 14
#     else:
#       count = 6
#
#     #Scan all blocks in area
#     for i in range(count):
#       #replace blocksim with block detector code
#       bs = BlockDet.BlockSim()
#       block = bs.process(loc,i)
#       ## possibly update block location here
#       ## block.setLocation(self.getCurrentLocation());
#       print "scanning block", i+1, ":", block.getColor(), block.getSize(), block.getLocation()
#
#       if loc == "storage":
#         if block.getSize() == "small":
#           self.nextAirBlock.append(block)
#         else:
#           self.nextSeaLandBlock.append(block)
#       elif loc == "land":
#         self.nextLandBlockLoc.append(block)
#       elif loc == "sea":
#         self.nextSeaBlockLoc.append(block)
#       # Update target location for next block
#
#       nextLoc = block.getLocation();
#       bLoc = [int(nextLoc[0]), int(nextLoc[1])]
#       bLoc[1] = bLoc[1] + 2; # change 2 to appropriate value
#
#       self.moveTo(self.getCurrentLocation(), bLoc)
    #end for
    #print self.nextAirBlock
    #print self.nextSeaLandBlock
  #end scanFirstTime

  # use this if dropping off blocks during scan
  def storageSimulator(self, loc):
    print "-- Using Block Detection Simulator"
    #Scan all 14 blocks in storage area
    for i in range(14):
      bs = BlockDet.BlockSim()
      block = bs.process(loc,i)
      self.storageSim.append(block)
    #end for
  #end scanStorageFirstTime

  # use this if dropping off blocks during scan
  def dropOffSimulator(self, loc):
    print "-- Using Zone Dection Simulator"
    print "Initiating", loc, "scan"
    for i in range(6):
      print "scanning", loc, "block location", i+1

      bs = BlockDet.BlockSim()
      blockLoc = bs.process(loc,i)

      if loc == "land":
        self.landBlockSim[blockLoc.getLocation()] = blockLoc.getColor()
        #self.nlbl[blockLoc.getColor()] = blockLoc.getLocation()
      elif loc == "sea":
        self.seaBlockSim[blockLoc.getLocation()] = blockLoc.getColor()
        #self.nsbl[blockLoc.getColor()] = blockLoc.getLocation()
    #end if
  #end scanLandorSeaFirstTime

def run(bot_loc, blobs, blocks, zones, waypoints, scPlanner, bot_state, qMove_nav):
  # TODO Handle shared data, start your process from here
  plan = Planner(bot_loc, blobs, blocks, zones, waypoints, scPlanner, bot_state, qMove_nav)
  #plan.test()
  plan.start()

if __name__ == "__main__":
  plan = Planner() #will fail... needs waypoints from map.
  plan.start()
<commit_msg>stub for micromove<commit_after>"""
Code for python planner
"""

import blockSim as BlockDet
import twoWayDict as twd
import navigation.nav as nav
from datetime import datetime
import comm.serial_interface as comm

class Planner:
  nextSeaLandBlock = [] #list of the next available sea or land block to pick up
  nextAirBlock = [] #list of the 2 air blocks in storage
  armList = [] #list of which arm contains what
  armID = [2, 0]  # armID[0] = right arm, armID[1] = left arm

  storageSim = []
  seaBlockSim = {}
  landBlockSim = {}
  airBlockSim = []

  nextSeaBlockLoc = []
  nextLandBlockLoc = []
  nextAirBlockLoc = []

  scannedSeaLocs = {}
  scannedLandLocs = {}
  colors = []
  #nsbl = twd.TwoWayDict()
  #nlbl = twd.TwoWayDict()

  def __init__(self, bot_loc, blobs, blocks, zones, waypoints, scPlanner, bot_state, qMove_nav):
    """Setup navigation class

    :param bot_loc: Shared dict updated with best-guess location of bot by localizer
    :param blocks: list of blocks
    :param zones:
    :param waypoints: Multiprocessing.Queue object for passing movement feedback to localizer from navigator
    :param si: Serial interface object for sending commands to low-level boards
    :param bot_state: Dict of information about the current state of the bot (ex macro/micro nav)
    :param qMove_nav: Multiprocessing.Queue object for passing movement commands to navigation (mostly from Planner)
    """
    # Store passed-in data
    self.bot_loc = bot_loc
    self.blobs = blobs
    self.blocks = blocks
    self.zones = zones
    self.waypoints = waypoints
    self.scPlanner = scPlanner
    self.bot_state = bot_state
    self.qMove_nav = qMove_nav

    self.armID[0] = comm.right_arm
    self.armID[1] = comm.left_arm

    self.nextSeaLandBlock = ["St01","St02","St03","St04","St05","St06","St07","St08","St09","St10","St11","St12","St13","St14"]
    self.nextAirBlock = []
    self.nextSeaBlockLoc = ["Se01","Se02","Se03","Se04","Se05","Se06"]
    self.nextLandBlockLoc = ["L01","L02","L03","L04","L05","L06"]
    self.colors = ["red", "blue", "green", "orange", "brown", "yellow"]

    for i in range(len(self.nextSeaLandBlock)):
      self.blocks[i] = 1
    for i in range(len(self.nextSeaBlockLoc)):
      self.zones[i] = 0
    for i in range(len(self.nextLandBlockLoc)):
      self.zones[i] = 0
    for i in range(len(self.colors)):
      self.scannedSeaLocs[self.colors[i]] = "empty"
      self.scannedLandLocs[self.colors[i]] = "empty"

  #get current location
  def getCurrentLocation(self):
    return self.bot_loc

  #more to the next block - use this if next block location is
  #handled by nav instead of planner
  def moveToNextBlock(self):
    print "Moving to Next Block"
    self.moveTo(self.getCurrentLocation(), "nextblock loc")
    #micro or macro???

  #move from start to end
  def moveToWayPoint(self, startLoc, endLoc):
    print "Moving from ", startLoc, " to ", endLoc, "--", self.waypoints[endLoc]
    x, y = self.waypoints[endLoc][1]
    theta = self.waypoints[endLoc][2]
    speed = self.waypoints[endLoc][3]
    self.bot_state["naving"] = True
    macro_m = nav.macro_move(x, y, theta, datetime.now())
    self.qMove_nav.put(macro_m)
    while self.bot_state["naving"] != False:
      continue

  def alignWithCenter(self, loc):

	#dist = 320-(x+w/2)
	#direction = 1
	#if dist < 0:
	#  direction = -1
	#print "Distance to center: ", dist, "pixels -- ", dist*0.0192, "inches --", dist*0.0192*1622/9.89,"revolutions"

	#self.micromove(dist, direction)

  def microMove(self, distance, direction):
    #print "Moving from ", startLoc, " to ", endLoc
	micro_m = nav.microMoveXY(distance, comm.default_speed * direction)
    self.qMove_nav.put(micro_m)

  def processSeaLand(self):
    armCount = 0
    armList = []
    print "+++++ +++++ +++++ +++++ +++++ +++++"
    for i in range(len(self.nextSeaLandBlock)):
      stID = self.nextSeaLandBlock[i];
      print "Processing: [", stID, self.waypoints[stID], "]"
      self.moveToWayPoint(self.getCurrentLocation(), stID)

      #get block from vision
      block = self.storageSim[i]
      #print "Processing: [", block.getColor(), block.getSize(), block.getLocation(), "]"

      #if the block is small, assign that to list of air blocks
      #continue / move to next block
      if block.getSize() == "small":
        self.nextAirBlock.append(block)
        continue

      #in order to pick up a block, first check if bot is centered
      #that code can exist in pickUpBlock().
      self.pickUpBlock(armCount) #arm 0 or 1.
      armList.append(block)
      armCount = armCount + 1;

      if armCount == 2:
        print "picked up 2 blocks"

		#when dropping blocks off, offset the center of the bot
		# about 0.5 from the center of the dropoff zone

        #Both arms contain sea blocks
        if armList[0].getSize() == "medium" and armList[1].getSize() == "medium":
          self.moveToWayPoint(self.getCurrentLocation(), "sea")

          goToNextDropOff(armList[0], "sea")
          self.placeBlock(0)

          goToNextDropOff(armList[1], "sea")
          self.placeBlock(1)

        #Both arms contain land blocks
        elif armList[0].getSize() == "large" and armList[1].getSize() == "large":
          self.moveToWayPoint(self.getCurrentLocation(), "land")

          goToNextDropOff(armList[0], "land")
          self.placeBlock(0)

          armId = goToNextDropOff(armList[1], "land")
          self.placeBlock(1)

        #One arm contains sea block and other contains land block
        elif armList[0].getSize() == "medium" and armList[1].getSize() == "large":
          self.moveToWayPoint(self.getCurrentLocation(), "sea")
          goToNextDropOff(armList, "sea")
          self.placeBlock(0)

          self.moveToWayPoint(self.getCurrentLocation(), "land")
          goToNextDropOff(armList, "land")
          self.placeBlock(1)

        #One arm contains land block and other contains sea block
        elif armList[0].getSize() == "large" and armList[1].getSize() == "medium":
          # even if the orders are different, first go to sea then land
          self.moveToWayPoint(self.getCurrentLocation(), "sea")
          goToNextDropOff(armList[1], "sea")
          self.placeBlock(1)

          self.moveToWayPoint(self.getCurrentLocation(), "land")
          goToNextDropOff(armList[0], "land")
          self.placeBlock(0)

        armCount = 0
        armList = []
        print "===== ===== ===== ===== ===== ===== ===== ===== ===== ====="
        self.moveToWayPoint(self.getCurrentLocation(), "storage")
      #end if
    #end for
  #end processSeaLand

  def processAir(self):
    for i in range(len(self.nextAirBlock)):
      block = self.nextAirBlock[i];
      print "Processing: [", block.getColor(), block.getSize(), block.getLocation(), "]"
      self.pickUpBlock(block.getLocation(), i);
    #end for

    print "Move to Ramp, up the ramp, to the drop-off"
    self.moveToWayPoint(self.getCurrentLocation(), "grnd2ramp") #normal speed
    self.moveToWayPoint(self.getCurrentLocation(), "lwrPlt") #ramp speed
    #self.moveToWayPint(self.getCurrentLocation(), "uprRamp") #normal speed
    self.moveToWayPoint(self.getCurrentLocation(), "air") #ramp speed

    print "Scan air drop-off"
    print "Drop Air Blocks"

  def getAvailableSeaDropOffs():
    availList = []
    for i in range(len(nextSeaBlockLoc)):
      if zones[nextSeaBlockLoc[i]] == 0:
        availList.append(nextSeaBlockLoc[i])
    return availList

  def getAvailableLandDropOffs():
    availList = []
    for i in range(len(nextLandBlockLoc)):
      if zones[nextLandBlockLoc[i]] == 0:
        availList.append(nextLandBlockLoc[i])
    return availList

  #go to the next sea dropoff zone
  #separate function to handle sea specific movements
  def goToNextSeaDropOff(self, block):
    # if seaDropLocList is empty, go to Se01
    # else, check if color of either block matches
    availList = getAvailableSeaDropOffs()
    blockColor = block.getColor()

    if scannedSeaLocs[blockColor] == "empty":
      #block location unknown
      for i in range(len(availList)):
        self.moveToWayPoint(self.getCurrentLocation(), availList[i])
        #get the color at waypoint
        color = seaBlockSim[availList[i]]
        if color == blockColor:
          #found color
          break
        else:
          scannedSeaLocs[color] = availList[i]

    else:
      self.moveToWayPoint(self.getCurrentLocation(), scannedSeaLocs[blockColor])

  #go to the next land dropoff zone
  #separate function to handle land specific movements
  def goToNextLandDropOff(self, block):
    availList = getAvailableLandDropOffs()
    blockColor = block.getColor()

    if scannedLandLocs[blockColor] == "empty":
      #block location unknown
      for i in range(len(availList)):
        self.moveToWayPoint(self.getCurrentLocation(), availList[i])
        #get the color at waypoint
        color = LandBlockSim[availList[i]]
        if color == blockColor:
          #found color
          break
        else:
          scannedLandLocs[color] = availList[i]

    else:
      self.moveToWayPoint(self.getCurrentLocation(), scannedLandLocs[blockColor])

  # pick up a block given armID
  def pickUpBlock(self, arm):
    armId = self.armID[arm]
    #self.moveTo(self.getCurrentLocation(), blockLoc)
    #print "Picking Up Block at ", blockLoc, "with Arm", armId
    #call vision to make sure we are centered on the block
    #if we are not centered, micromove
    self.scPlanner.gripperOpen(armId)
    self.scPlanner.armDown(armId)
    self.scPlanner.gripperClose(armId)
    self.scPlanner.armUp(armId)

  # place a block given armID
  def placeBlock(self, arm):
    armId = self.armID[arm]
    #self.moveTo(self.getCurrentLocation(), blockLoc)
    #print "Placing block from ", self.armID[arm], "at", blockLoc
    #call vision to make sure we are centered on the block
    #if we are not centered, micromove
    self.scPlanner.armDown(armId)
    self.scPlanner.gripperOpen(armId)
    self.scPlanner.armUp(armId)
    self.scPlanner.gripperClose(armId)

  #main
  def start(self):
    self.storageSimulator("storage") ##use when vision is not available
    self.dropOffSimulator("sea") ##use when vision is not available
    self.dropOffSimulator("land")
    print "Move to Storage Start"
    self.moveToWayPoint(self.getCurrentLocation(), "storage")
    #print "Scan Storage"
    #self.scanStorageFirstTime("storage") # BUG: This should not be hardcoded. Currently fails.
    #print "Move to Storage Start"
    #self.moveToWayPoint(self.getCurrentLocation(), "storage")
    print "***********************************************"
    print "********** Processing - Sea and Land **********"
    print "***********************************************"
    self.processSeaLand()
    print "**************************************"
    print "********** Processing - Air **********"
    print "**************************************"
    self.processAir()

  def test(self):
    print self.waypoints
    print len(self.waypoints)
    print self.waypoints["storage"]

  # Scan the given location for the first time
  # use this if scanning first then dropping blocks
  # def scanFirstTime(self, loc):
#     print "Scanning Storage"
#     print "Initiating Storage Scan"
#     print "updating list of sea, land and air blocks"
#
#     count = 0
#     if loc == "storage":
#       count = 14
#     else:
#       count = 6
#
#     #Scan all blocks in area
#     for i in range(count):
#       #replace blocksim with block detector code
#       bs = BlockDet.BlockSim()
#       block = bs.process(loc,i)
#       ## possibly update block location here
#       ## block.setLocation(self.getCurrentLocation());
#       print "scanning block", i+1, ":", block.getColor(), block.getSize(), block.getLocation()
#
#       if loc == "storage":
#         if block.getSize() == "small":
#           self.nextAirBlock.append(block)
#         else:
#           self.nextSeaLandBlock.append(block)
#       elif loc == "land":
#         self.nextLandBlockLoc.append(block)
#       elif loc == "sea":
#         self.nextSeaBlockLoc.append(block)
#       # Update target location for next block
#
#       nextLoc = block.getLocation();
#       bLoc = [int(nextLoc[0]), int(nextLoc[1])]
#       bLoc[1] = bLoc[1] + 2; # change 2 to appropriate value
#
#       self.moveTo(self.getCurrentLocation(), bLoc)
    #end for
    #print self.nextAirBlock
    #print self.nextSeaLandBlock
  #end scanFirstTime

  # use this if dropping off blocks during scan
  def storageSimulator(self, loc):
    print "-- Using Block Detection Simulator"
    #Scan all 14 blocks in storage area
    for i in range(14):
      bs = BlockDet.BlockSim()
      block = bs.process(loc,i)
      self.storageSim.append(block)
    #end for
  #end scanStorageFirstTime

  # use this if dropping off blocks during scan
  def dropOffSimulator(self, loc):
    print "-- Using Zone Dection Simulator"
    print "Initiating", loc, "scan"
    for i in range(6):
      print "scanning", loc, "block location", i+1

      bs = BlockDet.BlockSim()
      blockLoc = bs.process(loc,i)

      if loc == "land":
        self.landBlockSim[blockLoc.getLocation()] = blockLoc.getColor()
        #self.nlbl[blockLoc.getColor()] = blockLoc.getLocation()
      elif loc == "sea":
        self.seaBlockSim[blockLoc.getLocation()] = blockLoc.getColor()
        #self.nsbl[blockLoc.getColor()] = blockLoc.getLocation()
    #end if
  #end scanLandorSeaFirstTime

def run(bot_loc, blobs, blocks, zones, waypoints, scPlanner, bot_state, qMove_nav):
  # TODO Handle shared data, start your process from here
  plan = Planner(bot_loc, blobs, blocks, zones, waypoints, scPlanner, bot_state, qMove_nav)
  #plan.test()
  plan.start()

if __name__ == "__main__":
  plan = Planner() #will fail... needs waypoints from map.
  plan.start()
<|endoftext|>


<commit_before><commit_msg>test large integers plugin in numerical tower<commit_after><|endoftext|>


<commit_before># written by texelelf
#-# Adding a result pages, and NBT edit stuff
from pymclevel import TAG_Byte, TAG_Short, TAG_Int, TAG_Compound, TAG_List, TAG_String, TAG_Double, TAG_Float, TAG_Long, \
    TAG_Byte_Array, TAG_Int_Array
from pymclevel.box import BoundingBox
from albow import alert, ask
import ast
# Let import the stuff to save files.
from mcplatform import askSaveFile
from directories import getDocumentsFolder

RECORD_UNDO = False

displayName = "Find"

tagtypes = {"TAG_Byte": 0, "TAG_Short": 1, "TAG_Int": 2, "TAG_Compound": 3, "TAG_List": 4, "TAG_String": 5,
            "TAG_Double": 6, "TAG_Float": 7, "TAG_Long": 8, "TAG_Byte_Array": 9, "TAG_Int_Array": 10}

tagses = {0: TAG_Byte, 1: TAG_Short, 2: TAG_Int, 3: TAG_Compound, 4: TAG_List, 5: TAG_String,
          6: TAG_Double, 7: TAG_Float, 8: TAG_Long, 9: TAG_Byte_Array, 10: TAG_Int_Array, "Any": 11}

inputs = [(("Match by:", ("TileEntity", "Entity", "Block")),
           ("Match block type (for TileEntity searches):", False),
           ("Match block:", "blocktype"),
           ("Match block data:", True),
           ("Match tile entities (for Block searches):", False),
           ("\"None\" for Name or Value will match any tag's Name or Value respectively.", "label"),
           ("Match Tag Name:", ("string", "value=None")),
           ("Match Tag Value:", ("string", "value=None")),
           ("Case insensitive:", True),
           ("Match Tag Type:", tuple(("Any",)) + tuple(tagtypes.keys())),
           ("Operation:", ("Start New Search", "Dump Found Coordinates")),
           ("Options", "title")),

           (("Results", "title"), ("", ["NBTTree", {}, 0, False])), # [str name_of_widget_type, dict default_data, int page_to_goback, bool show_load_button]

           (("Documentation", "title"),
           ("This filter is designed to search for NBT in either Entities or TileEntities.\n"
            "It can also be used to search for blocks.\n\"Match by\" determines which type of object "
            "is prioritized during the search.\nEntites and TileEntities will search relatively quickly, "
            "while the speed of searching by Block will be directly proportional to the selection size (since every "
            "single block within the selection will be examined).\n"
            "All Entity searches will ignore the block settings; TileEntity searches will try to "
            "\"Match block type\" if checked, and Block searches will try to \"Match tile entity\" tags if "
            "checked.\nIt is faster to match TileEntity searches with a Block Type than vice versa.\nBlock "
            "matching can also optionally match block data, e.g. matching all torches, or only torches facing "
            "a specific direction.\n\"Start New Search\" will re-search through the selected volume, while \"Find Next\" "
            "will iterate through the search results of the previous search.", "label"))
]

tree = None # the tree widget
chunks = None # the chunks used to perform the search
bbox = None # the bouding box to search in
by = None # what is searched: Entities, TileEntities or blocs

def set_tree(t):
    global tree
    tree = t

# Use this method to overwrite the NBT tree default behaviour on mouse clicks
def nbttree_mouse_down(e):
    if e.num_clicks > 1:
        if tree.selected_item and tree.selected_item[3].startswith('(') and tree.selected_item[3].endswith(')'):
            s = ast.literal_eval(tree.selected_item[3])
            editor.mainViewport.cameraPosition = (s[0] + 0.5, s[1] + 2, s[2] - 1)
            editor.mainViewport.yaw = 0.0
            editor.mainViewport.pitch = 45.0

            newBox = BoundingBox(s, (1, 1, 1))
            editor.selectionTool.setSelection(newBox)
    tree.treeRow.__class__.mouse_down(tree.treeRow, e)

def get_chunks():
    return chunks

def get_box():
    return bbox

def get_by():
    return by

# Use this method to overwrite the NBT tree 'OK' button default behaviour
def nbt_ok_action():
    by = get_by()
    chunks = get_chunks()
    box = get_box()
    if by not in (trn._('TileEntity'), trn._('Entity')):
        return
    if chunks:
        for chunk, slices, point in chunks:
            if by == trn._('TileEntity'):
                for e in chunk.TileEntities:
                    x = e["x"].value
                    y = e["y"].value
                    z = e["z"].value
            elif by == trn._('Entity'):
                for e in chunk.Entities:
                    x = e["Pos"][0].value
                    y = e["Pos"][1].value
                    z = e["Pos"][2].value
            if (x, y, z) in box:
                chunk.dirty = True

try:
    search
except NameError:
    search = None

def FindTagS(nbtData, name, value, tagtype):
    if type(nbtData) is TAG_List or type(nbtData) is TAG_Compound:
        if name in nbtData.name or name == "":
            if value == "":
                if type(nbtData) is tagtype or tagtype == 11:
                    print "found in pre-area"
                    return True
        if type(nbtData) is TAG_List:
            list = True
        else:
            list = False

        for tag in range(0, len(nbtData)) if list else nbtData.keys():
            if type(nbtData[tag]) is TAG_Compound:
                if FindTagS(nbtData[tag], name, value, tagtype):
                    return True
            elif type(nbtData[tag]) is TAG_List:
                if FindTagS(nbtData[tag], name, value, tagtype):
                    return True
            else:
                if name in nbtData[tag].name or name == "":
                    if value in unicode(nbtData[tag].value):
                        if type(nbtData[tag]) is tagtype or tagtype == 11:
                            print "found in list/compound"
                            return True
        else:
            return False
    else:
        if name in nbtData.name or name == "":
            if value in unicode(nbtData.value):
                if type(nbtData[tag]) is tagtype or tagtype == 11:
                    print "found outside"
                    return True
    return False

def FindTagI(nbtData, name, value, tagtype):
    if type(nbtData) is TAG_List or type(nbtData) is TAG_Compound:
        if name in (u"%s"%nbtData.name).upper() or name == "":
            if value == "":
                if type(nbtData) is tagtype or tagtype == 11:
                    return True
        if type(nbtData) is TAG_List:
            list = True
        else:
            list = False

        for tag in range(0, len(nbtData)) if list else nbtData.keys():
            if type(nbtData[tag]) is TAG_Compound:
                if FindTagI(nbtData[tag], name, value, tagtype):
                    return True
            elif type(nbtData[tag]) is TAG_List:
                if FindTagI(nbtData[tag], name, value, tagtype):
                    return True
            else:
                if name in (u"%s"%nbtData[tag].name).upper() or name == "":
                    if value in unicode(nbtData[tag].value).upper():
                        if type(nbtData[tag]) is tagtype or tagtype == 11:
                            return True
        else:
            return False
    else:
        if name in (u"%s"%nbtData.name).upper() or name == "":
            if value in unicode(nbtData.value).upper():
                if type(nbtData[tag]) is tagtype or tagtype == 11:
                    return True
    return False

def FindTag(nbtData, name, value, tagtype, caseSensitive):
    if caseSensitive:
        return FindTagS(nbtData, name, value, tagtype)
    else:
        return FindTagI(nbtData, name, value, tagtype)

def perform(level, box, options):
    global search

    # Don't forget to 'globalize' these...
    global chunks
    global bbox
    global by
    bbox = box

    by = options["Match by:"]
    matchtype = options["Match block type (for TileEntity searches):"]
    matchblock = options["Match block:"]
    matchdata = options["Match block data:"]
    matchtile = options["Match tile entities (for Block searches):"]
    matchname = u"" if options["Match Tag Name:"] == "None" else unicode(options["Match Tag Name:"])
    matchval = u"" if options["Match Tag Value:"] == "None" else unicode(options["Match Tag Value:"])
    caseSensitive = not options["Case insensitive:"]
    matchtagtype = tagtypes.get(options["Match Tag Type:"], "Any")
    op = options["Operation:"]

    datas = []

    if not caseSensitive:
        matchname = matchname.upper()
        matchval = matchval.upper()

    if matchtile and matchname == "" and matchval == "":
        alert("\nInvalid Tag Name and Value; the present values will match every tag of the specified type.")

    if search is None or op == trn._("Start New Search") or op == trn._("Dump Found Coordinates"):
        search = []

    if not search:
        if by == trn._("Block"):
            for x in xrange(box.minx, box.maxx):
                for z in xrange(box.minz, box.maxz):
                    for y in xrange(box.miny, box.maxy):
                        block = level.blockAt(x, y, z)
                        data = level.blockDataAt(x, y, z)
                        if block == matchblock.ID and (not matchdata or data == matchblock.blockData):
                            pass
                        else:
                            continue
                        if matchtile:
                            tile = level.tileEntityAt(x, y, z)
                            if tile is not None:
                                if not FindTag(tile, matchname, matchval, tagses[matchtagtype], caseSensitive):
                                    continue
                            else:
                                continue
                        search.append((x, y, z))
                        datas.append(data)
        elif by == trn._("TileEntity"):
            chunks = []
            for (chunk, slices, point) in level.getChunkSlices(box):
                for e in chunk.TileEntities:
                    x = e["x"].value
                    y = e["y"].value
                    z = e["z"].value
                    if (x, y, z) in box:
                        if matchtype:
                            block = level.blockAt(x, y, z)
                            data = level.blockDataAt(x, y, z)
                            if block == matchblock.ID and (not matchdata or data == matchblock.blockData):
                                pass
                            else:
                                continue
                        if not FindTag(e, matchname, matchval, tagses[matchtagtype], caseSensitive):
                            continue

                        search.append((x, y, z))
                        datas.append(e)
                        chunks.append([chunk, slices, point])
        else:
            chunks = []
            for (chunk, slices, point) in level.getChunkSlices(box):
                for e in chunk.Entities:
                    x = e["Pos"][0].value
                    y = e["Pos"][1].value
                    z = e["Pos"][2].value
                    if (x, y, z) in box:
                        if FindTag(e, matchname, matchval, tagses[matchtagtype], caseSensitive):
                            search.append((x, y, z))
                            datas.append(e)
                            chunks.append([chunk, slices, point])
    if not search:
        alert("\nNo matching blocks/tile entities found")
    else:
        search.sort()
        if op == trn._("Dump Found Coordinates"):
            result = "\n".join("%d, %d, %d" % pos for pos in search)
            answer = ask(result, height=editor.height, colLabel="Matching Coordinates", responses=["Save", "OK"])
            if answer == "Save":
                fName = askSaveFile(getDocumentsFolder(), "Save to file...", "find.txt", 'Folder\0*.dat\0*.*\0\0', 'txt')
                if fName:
                    fData = "# MCEdit find output\n# Search options:\n# Match by: %s\n# Match block type: %s\n# Match block: %s\n# Match block data: %s\n# Match tile entities: %s\n# Match Tag Name:%s\n# Match Tag Value: %s\n# Case insensitive: %s\n# Match Tag Type: %s\n\n%s"%(by, matchtype, matchblock, matchdata, matchtile, matchname, matchval, caseSensitive, matchtagtype, result)
                    open(fName, 'w').write(fData)
        else:
            treeData = {}
            # To set tooltip text to the items the need it, use a dict: {"value": <item to be added to the tree>, "tooltipText": "Some text"}
            for i in range(len(search)):
                if by == trn._('Block'):
                    treeData[u"%s"%(search[i],)] = {"value": datas[i], "tooltipText": "Double-click to go to this item."}
                elif by == trn._('Entity'):
                    treeData[u"%s"%((datas[i]['Pos'][0].value, datas[i]['Pos'][1].value, datas[i]['Pos'][2].value),)] = {"value": datas[i], "tooltipText": "Double-click to go to this item."}
                else:
                    treeData[u"%s"%((datas[i]['x'].value, datas[i]['y'].value, datas[i]['z'].value),)] = {"value": datas[i], "tooltipText": "Double-click to go to this item."}
            inputs[1][1][1][1] = {'Data': treeData}
            options[""](inputs[1])
<commit_msg>Unactivated in-development capability.<commit_after># written by texelelf
#-# Adding a result pages, and NBT edit stuff
from pymclevel import TAG_Byte, TAG_Short, TAG_Int, TAG_Compound, TAG_List, TAG_String, TAG_Double, TAG_Float, TAG_Long, \
    TAG_Byte_Array, TAG_Int_Array
from pymclevel.box import BoundingBox
from albow import alert, ask
import ast
# Let import the stuff to save files.
from mcplatform import askSaveFile
from directories import getDocumentsFolder

# The RECODR_UNDO is not yet usable...
# RECORD_UNDO = False

displayName = "Find"

tagtypes = {"TAG_Byte": 0, "TAG_Short": 1, "TAG_Int": 2, "TAG_Compound": 3, "TAG_List": 4, "TAG_String": 5,
            "TAG_Double": 6, "TAG_Float": 7, "TAG_Long": 8, "TAG_Byte_Array": 9, "TAG_Int_Array": 10}

tagses = {0: TAG_Byte, 1: TAG_Short, 2: TAG_Int, 3: TAG_Compound, 4: TAG_List, 5: TAG_String,
          6: TAG_Double, 7: TAG_Float, 8: TAG_Long, 9: TAG_Byte_Array, 10: TAG_Int_Array, "Any": 11}

inputs = [(("Match by:", ("TileEntity", "Entity", "Block")),
           ("Match block type (for TileEntity searches):", False),
           ("Match block:", "blocktype"),
           ("Match block data:", True),
           ("Match tile entities (for Block searches):", False),
           ("\"None\" for Name or Value will match any tag's Name or Value respectively.", "label"),
           ("Match Tag Name:", ("string", "value=None")),
           ("Match Tag Value:", ("string", "value=None")),
           ("Case insensitive:", True),
           ("Match Tag Type:", tuple(("Any",)) + tuple(tagtypes.keys())),
           ("Operation:", ("Start New Search", "Dump Found Coordinates")),
           ("Options", "title")),

           (("Results", "title"), ("", ["NBTTree", {}, 0, False])), # [str name_of_widget_type, dict default_data, int page_to_goback, bool show_load_button]

           (("Documentation", "title"),
           ("This filter is designed to search for NBT in either Entities or TileEntities.\n"
            "It can also be used to search for blocks.\n\"Match by\" determines which type of object "
            "is prioritized during the search.\nEntites and TileEntities will search relatively quickly, "
            "while the speed of searching by Block will be directly proportional to the selection size (since every "
            "single block within the selection will be examined).\n"
            "All Entity searches will ignore the block settings; TileEntity searches will try to "
            "\"Match block type\" if checked, and Block searches will try to \"Match tile entity\" tags if "
            "checked.\nIt is faster to match TileEntity searches with a Block Type than vice versa.\nBlock "
            "matching can also optionally match block data, e.g. matching all torches, or only torches facing "
            "a specific direction.\n\"Start New Search\" will re-search through the selected volume, while \"Find Next\" "
            "will iterate through the search results of the previous search.", "label"))
]

tree = None # the tree widget
chunks = None # the chunks used to perform the search
bbox = None # the bouding box to search in
by = None # what is searched: Entities, TileEntities or blocs

def set_tree(t):
    global tree
    tree = t

# Use this method to overwrite the NBT tree default behaviour on mouse clicks
def nbttree_mouse_down(e):
    if e.num_clicks > 1:
        if tree.selected_item and tree.selected_item[3].startswith('(') and tree.selected_item[3].endswith(')'):
            s = ast.literal_eval(tree.selected_item[3])
            editor.mainViewport.cameraPosition = (s[0] + 0.5, s[1] + 2, s[2] - 1)
            editor.mainViewport.yaw = 0.0
            editor.mainViewport.pitch = 45.0

            newBox = BoundingBox(s, (1, 1, 1))
            editor.selectionTool.setSelection(newBox)
    tree.treeRow.__class__.mouse_down(tree.treeRow, e)

def get_chunks():
    return chunks

def get_box():
    return bbox

def get_by():
    return by

# Use this method to overwrite the NBT tree 'OK' button default behaviour
def nbt_ok_action():
    by = get_by()
    chunks = get_chunks()
    box = get_box()
    if by not in (trn._('TileEntity'), trn._('Entity')):
        return
    if chunks:
        for chunk, slices, point in chunks:
            if by == trn._('TileEntity'):
                for e in chunk.TileEntities:
                    x = e["x"].value
                    y = e["y"].value
                    z = e["z"].value
            elif by == trn._('Entity'):
                for e in chunk.Entities:
                    x = e["Pos"][0].value
                    y = e["Pos"][1].value
                    z = e["Pos"][2].value
            if (x, y, z) in box:
                chunk.dirty = True

try:
    search
except NameError:
    search = None

def FindTagS(nbtData, name, value, tagtype):
    if type(nbtData) is TAG_List or type(nbtData) is TAG_Compound:
        if name in nbtData.name or name == "":
            if value == "":
                if type(nbtData) is tagtype or tagtype == 11:
                    print "found in pre-area"
                    return True
        if type(nbtData) is TAG_List:
            list = True
        else:
            list = False

        for tag in range(0, len(nbtData)) if list else nbtData.keys():
            if type(nbtData[tag]) is TAG_Compound:
                if FindTagS(nbtData[tag], name, value, tagtype):
                    return True
            elif type(nbtData[tag]) is TAG_List:
                if FindTagS(nbtData[tag], name, value, tagtype):
                    return True
            else:
                if name in nbtData[tag].name or name == "":
                    if value in unicode(nbtData[tag].value):
                        if type(nbtData[tag]) is tagtype or tagtype == 11:
                            print "found in list/compound"
                            return True
        else:
            return False
    else:
        if name in nbtData.name or name == "":
            if value in unicode(nbtData.value):
                if type(nbtData[tag]) is tagtype or tagtype == 11:
                    print "found outside"
                    return True
    return False

def FindTagI(nbtData, name, value, tagtype):
    if type(nbtData) is TAG_List or type(nbtData) is TAG_Compound:
        if name in (u"%s"%nbtData.name).upper() or name == "":
            if value == "":
                if type(nbtData) is tagtype or tagtype == 11:
                    return True
        if type(nbtData) is TAG_List:
            list = True
        else:
            list = False

        for tag in range(0, len(nbtData)) if list else nbtData.keys():
            if type(nbtData[tag]) is TAG_Compound:
                if FindTagI(nbtData[tag], name, value, tagtype):
                    return True
            elif type(nbtData[tag]) is TAG_List:
                if FindTagI(nbtData[tag], name, value, tagtype):
                    return True
            else:
                if name in (u"%s"%nbtData[tag].name).upper() or name == "":
                    if value in unicode(nbtData[tag].value).upper():
                        if type(nbtData[tag]) is tagtype or tagtype == 11:
                            return True
        else:
            return False
    else:
        if name in (u"%s"%nbtData.name).upper() or name == "":
            if value in unicode(nbtData.value).upper():
                if type(nbtData[tag]) is tagtype or tagtype == 11:
                    return True
    return False

def FindTag(nbtData, name, value, tagtype, caseSensitive):
    if caseSensitive:
        return FindTagS(nbtData, name, value, tagtype)
    else:
        return FindTagI(nbtData, name, value, tagtype)

def perform(level, box, options):
    global search

    # Don't forget to 'globalize' these...
    global chunks
    global bbox
    global by
    bbox = box

    by = options["Match by:"]
    matchtype = options["Match block type (for TileEntity searches):"]
    matchblock = options["Match block:"]
    matchdata = options["Match block data:"]
    matchtile = options["Match tile entities (for Block searches):"]
    matchname = u"" if options["Match Tag Name:"] == "None" else unicode(options["Match Tag Name:"])
    matchval = u"" if options["Match Tag Value:"] == "None" else unicode(options["Match Tag Value:"])
    caseSensitive = not options["Case insensitive:"]
    matchtagtype = tagtypes.get(options["Match Tag Type:"], "Any")
    op = options["Operation:"]

    datas = []

    if not caseSensitive:
        matchname = matchname.upper()
        matchval = matchval.upper()

    if matchtile and matchname == "" and matchval == "":
        alert("\nInvalid Tag Name and Value; the present values will match every tag of the specified type.")

    if search is None or op == trn._("Start New Search") or op == trn._("Dump Found Coordinates"):
        search = []

    if not search:
        if by == trn._("Block"):
            for x in xrange(box.minx, box.maxx):
                for z in xrange(box.minz, box.maxz):
                    for y in xrange(box.miny, box.maxy):
                        block = level.blockAt(x, y, z)
                        data = level.blockDataAt(x, y, z)
                        if block == matchblock.ID and (not matchdata or data == matchblock.blockData):
                            pass
                        else:
                            continue
                        if matchtile:
                            tile = level.tileEntityAt(x, y, z)
                            if tile is not None:
                                if not FindTag(tile, matchname, matchval, tagses[matchtagtype], caseSensitive):
                                    continue
                            else:
                                continue
                        search.append((x, y, z))
                        datas.append(data)
        elif by == trn._("TileEntity"):
            chunks = []
            for (chunk, slices, point) in level.getChunkSlices(box):
                for e in chunk.TileEntities:
                    x = e["x"].value
                    y = e["y"].value
                    z = e["z"].value
                    if (x, y, z) in box:
                        if matchtype:
                            block = level.blockAt(x, y, z)
                            data = level.blockDataAt(x, y, z)
                            if block == matchblock.ID and (not matchdata or data == matchblock.blockData):
                                pass
                            else:
                                continue
                        if not FindTag(e, matchname, matchval, tagses[matchtagtype], caseSensitive):
                            continue

                        search.append((x, y, z))
                        datas.append(e)
                        chunks.append([chunk, slices, point])
        else:
            chunks = []
            for (chunk, slices, point) in level.getChunkSlices(box):
                for e in chunk.Entities:
                    x = e["Pos"][0].value
                    y = e["Pos"][1].value
                    z = e["Pos"][2].value
                    if (x, y, z) in box:
                        if FindTag(e, matchname, matchval, tagses[matchtagtype], caseSensitive):
                            search.append((x, y, z))
                            datas.append(e)
                            chunks.append([chunk, slices, point])
    if not search:
        alert("\nNo matching blocks/tile entities found")
    else:
        search.sort()
        if op == trn._("Dump Found Coordinates"):
            result = "\n".join("%d, %d, %d" % pos for pos in search)
            answer = ask(result, height=editor.height, colLabel="Matching Coordinates", responses=["Save", "OK"])
            if answer == "Save":
                fName = askSaveFile(getDocumentsFolder(), "Save to file...", "find.txt", 'Folder\0*.dat\0*.*\0\0', 'txt')
                if fName:
                    fData = "# MCEdit find output\n# Search options:\n# Match by: %s\n# Match block type: %s\n# Match block: %s\n# Match block data: %s\n# Match tile entities: %s\n# Match Tag Name:%s\n# Match Tag Value: %s\n# Case insensitive: %s\n# Match Tag Type: %s\n\n%s"%(by, matchtype, matchblock, matchdata, matchtile, matchname, matchval, caseSensitive, matchtagtype, result)
                    open(fName, 'w').write(fData)
        else:
            treeData = {}
            # To set tooltip text to the items the need it, use a dict: {"value": <item to be added to the tree>, "tooltipText": "Some text"}
            for i in range(len(search)):
                if by == trn._('Block'):
                    treeData[u"%s"%(search[i],)] = {"value": datas[i], "tooltipText": "Double-click to go to this item."}
                elif by == trn._('Entity'):
                    treeData[u"%s"%((datas[i]['Pos'][0].value, datas[i]['Pos'][1].value, datas[i]['Pos'][2].value),)] = {"value": datas[i], "tooltipText": "Double-click to go to this item."}
                else:
                    treeData[u"%s"%((datas[i]['x'].value, datas[i]['y'].value, datas[i]['z'].value),)] = {"value": datas[i], "tooltipText": "Double-click to go to this item."}
            inputs[1][1][1][1] = {'Data': treeData}
            options[""](inputs[1])
<|endoftext|>


<commit_before><commit_msg>New function to copy WGS sets from ftp<commit_after><|endoftext|>


<commit_before># coding=utf-8
"""
Admin panel for tags
"""
from __future__ import absolute_import, print_function, unicode_literals

import logging

import sqlalchemy as sa
from flask import current_app, flash, redirect, render_template, request
from sqlalchemy.sql import functions

from abilian.core.entities import Entity
from abilian.core.models.tag import Tag, entity_tag_tbl
from abilian.i18n import _, _l, _n
from abilian.services.indexing.service import index_update
from abilian.web import url_for
from abilian.web.admin import AdminPanel
from abilian.web.views import ObjectEdit
from abilian.web.views.base import View

from .forms import TagForm

logger = logging.getLogger(__name__)

_OBJ_COUNT = functions.count(entity_tag_tbl.c.entity_id).label('obj_count')

def get_entities_for_reindex(tags):
    """
    Collect entities for theses tags.
    """
    if isinstance(tags, Tag):
        tags = (tags,)

    session = current_app.db.session()
    indexing = current_app.services['indexing']
    tbl = Entity.__table__
    tag_ids = [t.id for t in tags]
    query = sa.sql \
        .select([tbl.c.entity_type, tbl.c.id]) \
        .select_from(tbl.join(entity_tag_tbl,
                              entity_tag_tbl.c.entity_id == tbl.c.id)) \
        .where(entity_tag_tbl.c.tag_id.in_(tag_ids))

    entities = set()

    with session.no_autoflush:
        for entity_type, entity_id in session.execute(query):
            if entity_type not in indexing.adapted:
                logger.debug('%r is not indexed, skipping', entity_type)

            item = ('changed', entity_type, entity_id, ())
            entities.add(item)

    return entities

def schedule_entities_reindex(entities):
    """
    :param entities: as returned by :func:`get_entities_for_reindex`
    """
    entities = [(e[0], e[1], e[2], dict(e[3])) for e in entities]
    return index_update.apply_async(kwargs=dict(
        index='default', items=entities))

class NSView(View):
    """
    View a Namespace
    """

    def __init__(self, view_endpoint, *args, **kwargs):
        super(NSView, self).__init__(*args, **kwargs)
        self.__selected_tags = None
        self.view_endpoint = view_endpoint

    def prepare_args(self, args, kwargs):
        self.ns = kwargs.get('ns')
        self.form_errors = {}
        return args, kwargs

    def get(self, ns):
        tags = Tag.query \
            .filter(Tag.ns == ns) \
            .outerjoin(entity_tag_tbl, entity_tag_tbl.c.tag_id == Tag.id) \
            .add_column(_OBJ_COUNT) \
            .group_by(Tag) \
            .order_by(sa.sql.func.lower(Tag.label))

        # get a list of rows instead of (Tag, count) tuples
        tags = list(tags.session.execute(tags))
        return render_template(
            'admin/tags_ns.html',
            ns=ns,
            tags=tags,
            errors=self.form_errors,
            merge_to=request.form.get('merge_to', default='__None__', type=int),
            selected_tags=set(t.id for t in self._get_selected_tags()),)

    def redirect_to_view(self):
        return redirect(url_for('.tags_ns', ns=self.ns))

    def post(self, ns):
        data = request.form
        action = data.get('__action')

        if action == u'delete':
            return self.do_delete()
        elif action == u'merge':
            return self.do_merge()

        else:
            flash(_(u'Unknown action'))
            self.get(self.ns)

    def _get_selected_tags(self):
        if self.__selected_tags is None:

            tag_ids = request.form.getlist('selected', type=int)
            if not tag_ids:
                self.__selected_tags = []
            else:
                self.__selected_tags = Tag.query \
                    .filter(Tag.ns == self.ns, Tag.id.in_(tag_ids)) \
                    .all()
        return self.__selected_tags

    def do_delete(self):
        data = request.form
        confirm = data.get('confirm_delete', False, type=bool)

        if not confirm:
            flash(_(u'Please fix the error(s) below'), 'error')
            self.form_errors['confirm_delete'] = _(u'Must be checked to ensure you '
                                                   u'intent to delete these tags')
            return self.get(self.ns)

        session = current_app.db.session()
        tags = self._get_selected_tags()

        if not tags:
            flash(_(u'No action performed: no tags selected'), 'warning')
            return self.redirect_to_view()

        count = len(tags)
        entities_to_reindex = get_entities_for_reindex(tags)
        success_message = _n(u'%(tag)s deleted',
                             u'%(num)d tags deleted:\n%(tags)s',
                             count,
                             tag=tags[0].label,
                             tags=', '.join(t.label for t in tags))
        map(session.delete, tags)
        session.commit()
        flash(success_message)
        schedule_entities_reindex(entities_to_reindex)
        return self.redirect_to_view()

    def do_merge(self):
        target_id = request.form.get('merge_to', type=int)

        if not target_id:
            flash(_(u'You must select a target tag to merge to'), 'error')
            return self.get(self.ns)

        target = Tag.query \
            .filter(Tag.ns == self.ns, Tag.id == target_id) \
            .scalar()

        if not target:
            flash(_(u'Target tag not found, no action performed'), 'error')
            return self.get(self.ns)

        merge_from = set(self._get_selected_tags())

        if target in merge_from:
            merge_from.remove(target)

        if not merge_from:
            flash(_(u'No tag selected for merging'), 'warning')
            return self.get(self.ns)

        session = current_app.db.session()
        merge_from_ids = [t.id for t in merge_from]
        tbl = entity_tag_tbl
        entities_to_reindex = get_entities_for_reindex(merge_from)
        already_tagged = sa.sql \
            .select([tbl.c.entity_id]) \
            .where(tbl.c.tag_id == target.id)
        del_dup = tbl.delete() \
            .where(sa.sql.and_(tbl.c.tag_id.in_(merge_from_ids),
                               tbl.c.entity_id.in_(already_tagged)))
        session.execute(del_dup)
        update = tbl.update() \
            .where(tbl.c.tag_id.in_(merge_from_ids)) \
            .values(tag_id=target.id)
        session.execute(update)
        map(session.delete, merge_from)
        session.commit()
        schedule_entities_reindex(entities_to_reindex)
        return self.redirect_to_view()

class BaseTagView(object):
    """
    Mixin for tag views
    """
    Model = Tag
    Form = TagForm

    def __init__(self, *args, **kwargs):
        super(BaseTagView, self).__init__(*args, **kwargs)
        self.extension = current_app.extensions['tags']

    def prepare_args(self, args, kwargs):
        self.ns = kwargs.pop('ns')
        return super(BaseTagView, self).prepare_args(args, kwargs)

    def view_url(self):
        return url_for('.tags_ns', ns=self.ns)

    index_url = view_url

class TagEdit(BaseTagView, ObjectEdit):
    _message_success = _l(u'Tag edited')

    def after_populate_obj(self):
        session = sa.orm.object_session(self.obj)
        self.has_changes = self.obj in (session.dirty | session.deleted)
        if self.has_changes:
            # since the tag may be in pending-delete, we must collect them before
            # flush
            self._entities_to_reindex = get_entities_for_reindex(self.obj)

    def commit_success(self):
        if not (self.has_changes and self._entities_to_reindex):
            return

        schedule_entities_reindex(self._entities_to_reindex)

class TagPanel(AdminPanel):
    """
    Tags administration
    """
    id = 'tags'
    label = _l(u'Tags')
    icon = 'tags'

    def get(self):
        obj_count = sa.sql \
            .select([Tag.ns,
                     functions.count(entity_tag_tbl.c.entity_id).label('obj_count')]) \
            .select_from(Tag.__table__.join(entity_tag_tbl)) \
            .group_by(Tag.ns) \
            .alias()

        ns_query = sa.sql \
            .select([Tag.ns,
                     functions.count(Tag.id).label('tag_count'),
                     obj_count.c.obj_count],
                    from_obj=[Tag.__table__.outerjoin(obj_count, Tag.ns == obj_count.c.ns)]) \
            .group_by(Tag.ns, obj_count.c.obj_count) \
            .order_by(Tag.ns)

        session = current_app.db.session()
        namespaces = session.execute(ns_query)

        return render_template('admin/tags.html', namespaces=namespaces)

    def install_additional_rules(self, add_url_rule):
        panel_endpoint = '.' + self.id
        ns_base = '/<string:ns>/'
        add_url_rule(
            ns_base,
            endpoint='ns',
            view_func=NSView.as_view(
                'ns', view_endpoint=panel_endpoint))

        tag_base = ns_base + '<int:object_id>/'
        add_url_rule(
            tag_base,
            endpoint='tag_edit',
            view_func=TagEdit.as_view(
                'tag_edit', view_endpoint=panel_endpoint))

        add_url_rule(
            tag_base + 'delete',
            endpoint='tag_delete',
            view_func=TagEdit.as_view(
                'tag_delete', view_endpoint=panel_endpoint))
<commit_msg>Type hinting.<commit_after># coding=utf-8
"""
Admin panel for tags
"""
from __future__ import absolute_import, print_function, unicode_literals

import logging

import sqlalchemy as sa
from flask import current_app, flash, redirect, render_template, request
from sqlalchemy.sql import functions

from abilian.core.entities import Entity
from abilian.core.models.tag import Tag, entity_tag_tbl
from abilian.i18n import _, _l, _n
from abilian.services.indexing.service import index_update
from abilian.web import url_for
from abilian.web.admin import AdminPanel
from abilian.web.views import ObjectEdit
from abilian.web.views.base import View

from .forms import TagForm

logger = logging.getLogger(__name__)

_OBJ_COUNT = functions.count(entity_tag_tbl.c.entity_id).label('obj_count')

def get_entities_for_reindex(tags):
    """
    Collect entities for theses tags.
    """
    if isinstance(tags, Tag):
        tags = (tags,)

    session = current_app.db.session()
    indexing = current_app.services['indexing']
    tbl = Entity.__table__
    tag_ids = [t.id for t in tags]
    query = sa.sql \
        .select([tbl.c.entity_type, tbl.c.id]) \
        .select_from(tbl.join(entity_tag_tbl,
                              entity_tag_tbl.c.entity_id == tbl.c.id)) \
        .where(entity_tag_tbl.c.tag_id.in_(tag_ids))

    entities = set()

    with session.no_autoflush:
        for entity_type, entity_id in session.execute(query):
            if entity_type not in indexing.adapted:
                logger.debug('%r is not indexed, skipping', entity_type)

            item = ('changed', entity_type, entity_id, ())
            entities.add(item)

    return entities

def schedule_entities_reindex(entities):
    """
    :param entities: as returned by :func:`get_entities_for_reindex`
    """
    entities = [(e[0], e[1], e[2], dict(e[3])) for e in entities]
    return index_update.apply_async(kwargs=dict(
        index='default', items=entities))

class NSView(View):
    """
    View a Namespace
    """

    def __init__(self, view_endpoint, *args, **kwargs):
        super(NSView, self).__init__(*args, **kwargs)
        self.__selected_tags = None
        self.view_endpoint = view_endpoint

    def prepare_args(self, args, kwargs):
        self.ns = kwargs.get('ns')
        self.form_errors = {}
        return args, kwargs

    def get(self, ns):
        tags = Tag.query \
            .filter(Tag.ns == ns) \
            .outerjoin(entity_tag_tbl, entity_tag_tbl.c.tag_id == Tag.id) \
            .add_column(_OBJ_COUNT) \
            .group_by(Tag) \
            .order_by(sa.sql.func.lower(Tag.label))

        # get a list of rows instead of (Tag, count) tuples
        tags = list(tags.session.execute(tags))
        return render_template(
            'admin/tags_ns.html',
            ns=ns,
            tags=tags,
            errors=self.form_errors,
            merge_to=request.form.get('merge_to', default='__None__', type=int),
            selected_tags=set(t.id for t in self._get_selected_tags()),)

    def redirect_to_view(self):
        return redirect(url_for('.tags_ns', ns=self.ns))

    def post(self, ns):
        data = request.form
        action = data.get('__action')

        if action == u'delete':
            return self.do_delete()
        elif action == u'merge':
            return self.do_merge()

        else:
            flash(_(u'Unknown action'))
            self.get(self.ns)

    def _get_selected_tags(self):
        """
        :rtype: List[Tag]
        """
        if self.__selected_tags is None:
            tag_ids = request.form.getlist('selected', type=int)
            if not tag_ids:
                self.__selected_tags = []
            else:
                self.__selected_tags = Tag.query \
                    .filter(Tag.ns == self.ns, Tag.id.in_(tag_ids)) \
                    .all()

        return self.__selected_tags

    def do_delete(self):
        data = request.form
        confirm = data.get('confirm_delete', False, type=bool)

        if not confirm:
            flash(_(u'Please fix the error(s) below'), 'error')
            self.form_errors['confirm_delete'] = _(u'Must be checked to ensure you '
                                                   u'intent to delete these tags')
            return self.get(self.ns)

        session = current_app.db.session()
        tags = self._get_selected_tags()

        if not tags:
            flash(_(u'No action performed: no tags selected'), 'warning')
            return self.redirect_to_view()

        count = len(tags)
        entities_to_reindex = get_entities_for_reindex(tags)
        success_message = _n(u'%(tag)s deleted',
                             u'%(num)d tags deleted:\n%(tags)s',
                             count,
                             tag=tags[0].label,
                             tags=', '.join(t.label for t in tags))
        map(session.delete, tags)
        session.commit()
        flash(success_message)
        schedule_entities_reindex(entities_to_reindex)
        return self.redirect_to_view()

    def do_merge(self):
        target_id = request.form.get('merge_to', type=int)

        if not target_id:
            flash(_(u'You must select a target tag to merge to'), 'error')
            return self.get(self.ns)

        target = Tag.query \
            .filter(Tag.ns == self.ns, Tag.id == target_id) \
            .scalar()

        if not target:
            flash(_(u'Target tag not found, no action performed'), 'error')
            return self.get(self.ns)

        merge_from = set(self._get_selected_tags())

        if target in merge_from:
            merge_from.remove(target)

        if not merge_from:
            flash(_(u'No tag selected for merging'), 'warning')
            return self.get(self.ns)

        session = current_app.db.session()
        merge_from_ids = [t.id for t in merge_from]
        tbl = entity_tag_tbl
        entities_to_reindex = get_entities_for_reindex(merge_from)
        already_tagged = sa.sql \
            .select([tbl.c.entity_id]) \
            .where(tbl.c.tag_id == target.id)
        del_dup = tbl.delete() \
            .where(sa.sql.and_(tbl.c.tag_id.in_(merge_from_ids),
                               tbl.c.entity_id.in_(already_tagged)))
        session.execute(del_dup)
        update = tbl.update() \
            .where(tbl.c.tag_id.in_(merge_from_ids)) \
            .values(tag_id=target.id)
        session.execute(update)
        map(session.delete, merge_from)
        session.commit()
        schedule_entities_reindex(entities_to_reindex)
        return self.redirect_to_view()

class BaseTagView(object):
    """
    Mixin for tag views
    """
    Model = Tag
    Form = TagForm

    def __init__(self, *args, **kwargs):
        super(BaseTagView, self).__init__(*args, **kwargs)
        self.extension = current_app.extensions['tags']

    def prepare_args(self, args, kwargs):
        self.ns = kwargs.pop('ns')
        return super(BaseTagView, self).prepare_args(args, kwargs)

    def view_url(self):
        return url_for('.tags_ns', ns=self.ns)

    index_url = view_url

class TagEdit(BaseTagView, ObjectEdit):
    _message_success = _l(u'Tag edited')

    def after_populate_obj(self):
        session = sa.orm.object_session(self.obj)
        self.has_changes = self.obj in (session.dirty | session.deleted)
        if self.has_changes:
            # since the tag may be in pending-delete, we must collect them before
            # flush
            self._entities_to_reindex = get_entities_for_reindex(self.obj)

    def commit_success(self):
        if not (self.has_changes and self._entities_to_reindex):
            return

        schedule_entities_reindex(self._entities_to_reindex)

class TagPanel(AdminPanel):
    """
    Tags administration
    """
    id = 'tags'
    label = _l(u'Tags')
    icon = 'tags'

    def get(self):
        obj_count = sa.sql \
            .select([Tag.ns,
                     functions.count(entity_tag_tbl.c.entity_id).label('obj_count')]) \
            .select_from(Tag.__table__.join(entity_tag_tbl)) \
            .group_by(Tag.ns) \
            .alias()

        ns_query = sa.sql \
            .select([Tag.ns,
                     functions.count(Tag.id).label('tag_count'),
                     obj_count.c.obj_count],
                    from_obj=[Tag.__table__.outerjoin(obj_count, Tag.ns == obj_count.c.ns)]) \
            .group_by(Tag.ns, obj_count.c.obj_count) \
            .order_by(Tag.ns)

        session = current_app.db.session()
        namespaces = session.execute(ns_query)

        return render_template('admin/tags.html', namespaces=namespaces)

    def install_additional_rules(self, add_url_rule):
        panel_endpoint = '.' + self.id
        ns_base = '/<string:ns>/'
        add_url_rule(
            ns_base,
            endpoint='ns',
            view_func=NSView.as_view(
                'ns', view_endpoint=panel_endpoint))

        tag_base = ns_base + '<int:object_id>/'
        add_url_rule(
            tag_base,
            endpoint='tag_edit',
            view_func=TagEdit.as_view(
                'tag_edit', view_endpoint=panel_endpoint))

        add_url_rule(
            tag_base + 'delete',
            endpoint='tag_delete',
            view_func=TagEdit.as_view(
                'tag_delete', view_endpoint=panel_endpoint))
<|endoftext|>


<commit_before>

__author__ = "Ryan Faulkner"
__date__ = "July 27th, 2012"
__license__ = "GPL (version 2 or later)"

from user_metrics.config import logging
from os import getpid

from dateutil.parser import parse as date_parse
import user_metric as um
from user_metrics.etl.aggregator import weighted_rate, decorator_builder, \
    build_numpy_op_agg, build_agg_meta
from user_metrics.metrics import query_mod
from numpy import median, min, max
import user_metrics.utils.multiprocessing_wrapper as mpw

# Constants for threshold events
LAST_EDIT = -1
REGISTRATION = 0

class TimeToThreshold(um.UserMetric):
    """
        Produces an integer value representing the number of minutes taken to
        reach a threshold.

          `https://meta.wikimedia.org/wiki/Research:Metrics/time_to_threshold`

        As a UserMetric type this class utilizes the process() function
        attribute to produce an internal list of metrics by user handle
        (typically ID but user names may also be specified). The execution of
        process() produces a nested list that stores in each element:

            * User ID
            * Time difference in minutes between events

        Below is an example of how we can generate the time taken to reach the
        first edit and the last edit from registration : ::

            >>> import user_metrics.metrics.time_to_threshold as t
            >>> t.TimeToThreshold(t.TimeToThreshold.EditCountThreshold,
                first_edit=0, threshold_edit=1).process([13234584]).
                    __iter__().next()
            [13234584, 3318]
            >>> t.TimeToThreshold(t.TimeToThreshold.EditCountThreshold,
                first_edit=0, threshold_edit=-1).process([13234584]).
                    __iter__().next()
            [13234584, 18906]

        If the termination event never occurs the number of minutes returned
        is -1.
    """

    # Structure that defines parameters for TimeToThreshold class
    _param_types = {
        'init':
        {
                'first_edit': ['int',
                               'Event that initiates measurement period.',
                               REGISTRATION],
                'threshold_edit': ['int', 'Threshold event.', 1],
        },
        'process': {},
    }

    # Define the metrics data model meta
    _data_model_meta = {
        'id_fields': [0],
        'date_fields': [],
        'float_fields': [],
        'integer_fields': [1],
        'boolean_fields': [],
    }

    _agg_indices = {
        'list_sum_indices': _data_model_meta['integer_fields'] +
        _data_model_meta['float_fields'],
    }

    @um.pre_metrics_init
    def __init__(self, **kwargs):
        super(TimeToThreshold, self).__init__(**kwargs)

    @staticmethod
    def header():
        return ['user_id', 'minutes_diff']

    @um.UserMetric.pre_process_metric_call
    def process(self, users, **kwargs):
        """ Wrapper for specific threshold objects """

        args = self._pack_params()
        self._results = mpw.build_thread_pool(users, _process_help,
                                              self.k_, args)

        return self

def _process_help(args):
    """
        First determine if the user has made an adequate number of
        edits.  If so, compute the number of minutes that passed
        between the Nth and Mth edit.

            - Parameters:
                - **user_handle** - List(int).  List of user ids.
                - **first_edit** - Integer.  The numeric value of
                    the first edit from which to measure the threshold.
                - **threshold_edit** - Integer.  The numeric value of
                    the threshold edit from which to measure the
                    threshold
    """

    # Unpack args
    state = args[1]
    users = args[0]

    thread_args = um.UserMetric._unpack_params(state)

    # Log progress
    if thread_args.log_:
        logging.debug(__name__ + '::Computing live account. (PID = %s)' %
                                 getpid())
    minutes_to_threshold = list()

    # For each user gather their revisions and produce a time diff
    for user in users:
        revs = query_mod.\
            time_to_threshold_revs_query(user, thread_args.project, None)
        revs = [rev[0] for rev in revs]
        minutes_to_threshold.append(
            [user, get_minute_diff_result(revs,
                thread_args.threshold_edit,
                thread_args.first_edit)])

    return minutes_to_threshold

def get_minute_diff_result(results, first, threshold):
    """
        Helper method.  This computes the minutes
        to threshold for the timestamp results.

            - Parameters:
                - **results** - list.  list of revision records with
                    timestamp for a given user.
    """
    if threshold == REGISTRATION and len(results):
        dat_obj_end = date_parse(results[0])
    elif threshold == LAST_EDIT and len(results):
        dat_obj_end = date_parse(results[len(results) - 1])
    elif threshold < len(results):
        dat_obj_end = date_parse(results[threshold])
    else:
        return -1

    if first == REGISTRATION and len(results) > 0:
        dat_obj_start = date_parse(results[0])
    elif first == LAST_EDIT and len(results):
        dat_obj_start = date_parse(results[len(results) - 1])
    elif first < len(results):
        dat_obj_start = date_parse(results[first])
    else:
        return -1

    time_diff = dat_obj_end - dat_obj_start
    return int(time_diff.seconds / 60) + abs(time_diff.days) * 24

# ==========================
# DEFINE METRIC AGGREGATORS
# ==========================

# Build "average" aggregator
ttt_avg_agg = weighted_rate
ttt_avg_agg = decorator_builder(TimeToThreshold.header())(ttt_avg_agg)

setattr(ttt_avg_agg, um.METRIC_AGG_METHOD_FLAG, True)
setattr(ttt_avg_agg, um.METRIC_AGG_METHOD_NAME, 'ttt_avg_agg')
setattr(ttt_avg_agg, um.METRIC_AGG_METHOD_HEAD, ['total_users',
                                                 'total_weight',
                                                 'average'])
setattr(ttt_avg_agg, um.METRIC_AGG_METHOD_KWARGS, {'val_idx': 1})

metric_header = TimeToThreshold.header()

field_prefixes = {
    'time_diff_': 1,
}

# Build "dist" decorator
op_list = [median, min, max]
ttt_stats_agg = build_numpy_op_agg(build_agg_meta(op_list, field_prefixes),
                                   metric_header,
                                   'ttt_stats_agg')

if __name__ == "__main__":
    for i in TimeToThreshold(threshold_type_class='edit_count_threshold',
                             first_edit=0,
                             threshold_edit=1).process([13234584]):
        print i
<commit_msg>fix. param_type types.<commit_after>

__author__ = "Ryan Faulkner"
__date__ = "July 27th, 2012"
__license__ = "GPL (version 2 or later)"

from user_metrics.config import logging
from os import getpid

from dateutil.parser import parse as date_parse
import user_metric as um
from user_metrics.etl.aggregator import weighted_rate, decorator_builder, \
    build_numpy_op_agg, build_agg_meta
from user_metrics.metrics import query_mod
from numpy import median, min, max
import user_metrics.utils.multiprocessing_wrapper as mpw

# Constants for threshold events
LAST_EDIT = -1
REGISTRATION = 0

class TimeToThreshold(um.UserMetric):
    """
        Produces an integer value representing the number of minutes taken to
        reach a threshold.

          `https://meta.wikimedia.org/wiki/Research:Metrics/time_to_threshold`

        As a UserMetric type this class utilizes the process() function
        attribute to produce an internal list of metrics by user handle
        (typically ID but user names may also be specified). The execution of
        process() produces a nested list that stores in each element:

            * User ID
            * Time difference in minutes between events

        Below is an example of how we can generate the time taken to reach the
        first edit and the last edit from registration : ::

            >>> import user_metrics.metrics.time_to_threshold as t
            >>> t.TimeToThreshold(t.TimeToThreshold.EditCountThreshold,
                first_edit=0, threshold_edit=1).process([13234584]).
                    __iter__().next()
            [13234584, 3318]
            >>> t.TimeToThreshold(t.TimeToThreshold.EditCountThreshold,
                first_edit=0, threshold_edit=-1).process([13234584]).
                    __iter__().next()
            [13234584, 18906]

        If the termination event never occurs the number of minutes returned
        is -1.
    """

    # Structure that defines parameters for TimeToThreshold class
    _param_types = {
        'init':
        {
                'first_edit': [int,
                               'Event that initiates measurement period.',
                               REGISTRATION],
                'threshold_edit': [int, 'Threshold event.', 1],
        },
        'process': {},
    }

    # Define the metrics data model meta
    _data_model_meta = {
        'id_fields': [0],
        'date_fields': [],
        'float_fields': [],
        'integer_fields': [1],
        'boolean_fields': [],
    }

    _agg_indices = {
        'list_sum_indices': _data_model_meta['integer_fields'] +
        _data_model_meta['float_fields'],
    }

    @um.pre_metrics_init
    def __init__(self, **kwargs):
        super(TimeToThreshold, self).__init__(**kwargs)

    @staticmethod
    def header():
        return ['user_id', 'minutes_diff']

    @um.UserMetric.pre_process_metric_call
    def process(self, users, **kwargs):
        """ Wrapper for specific threshold objects """

        args = self._pack_params()
        self._results = mpw.build_thread_pool(users, _process_help,
                                              self.k_, args)

        return self

def _process_help(args):
    """
        First determine if the user has made an adequate number of
        edits.  If so, compute the number of minutes that passed
        between the Nth and Mth edit.

            - Parameters:
                - **user_handle** - List(int).  List of user ids.
                - **first_edit** - Integer.  The numeric value of
                    the first edit from which to measure the threshold.
                - **threshold_edit** - Integer.  The numeric value of
                    the threshold edit from which to measure the
                    threshold
    """

    # Unpack args
    state = args[1]
    users = args[0]

    thread_args = um.UserMetric._unpack_params(state)

    # Log progress
    if thread_args.log_:
        logging.debug(__name__ + '::Computing Time to threshold. (PID = %s)' %
                                 getpid())

    minutes_to_threshold = list()

    # For each user gather their revisions and produce a time diff
    for user in users:
        revs = query_mod.\
            time_to_threshold_revs_query(user, thread_args.project, None)
        revs = [rev[0] for rev in revs]
        minutes_to_threshold.append(
            [user, get_minute_diff_result(revs,
                                          thread_args.threshold_edit,
                                          thread_args.first_edit)])

    return minutes_to_threshold

def get_minute_diff_result(results, first, threshold):
    """
        Helper method.  This computes the minutes
        to threshold for the timestamp results.

            - Parameters:
                - **results** - list.  list of revision records with
                    timestamp for a given user.
    """
    if threshold == REGISTRATION and len(results):
        dat_obj_end = date_parse(results[0])
    elif threshold == LAST_EDIT and len(results):
        dat_obj_end = date_parse(results[len(results) - 1])
    elif threshold < len(results):
        dat_obj_end = date_parse(results[threshold])
    else:
        return -1

    if first == REGISTRATION and len(results) > 0:
        dat_obj_start = date_parse(results[0])
    elif first == LAST_EDIT and len(results):
        dat_obj_start = date_parse(results[len(results) - 1])
    elif first < len(results):
        dat_obj_start = date_parse(results[first])
    else:
        return -1

    time_diff = dat_obj_end - dat_obj_start
    return int(time_diff.seconds / 60) + abs(time_diff.days) * 24

# ==========================
# DEFINE METRIC AGGREGATORS
# ==========================

# Build "average" aggregator
ttt_avg_agg = weighted_rate
ttt_avg_agg = decorator_builder(TimeToThreshold.header())(ttt_avg_agg)

setattr(ttt_avg_agg, um.METRIC_AGG_METHOD_FLAG, True)
setattr(ttt_avg_agg, um.METRIC_AGG_METHOD_NAME, 'ttt_avg_agg')
setattr(ttt_avg_agg, um.METRIC_AGG_METHOD_HEAD, ['total_users',
                                                 'total_weight',
                                                 'average'])
setattr(ttt_avg_agg, um.METRIC_AGG_METHOD_KWARGS, {'val_idx': 1})

metric_header = TimeToThreshold.header()

field_prefixes = {
    'time_diff_': 1,
}

# Build "dist" decorator
op_list = [median, min, max]
ttt_stats_agg = build_numpy_op_agg(build_agg_meta(op_list, field_prefixes),
                                   metric_header,
                                   'ttt_stats_agg')

if __name__ == "__main__":
    for i in TimeToThreshold(threshold_type_class='edit_count_threshold',
                             first_edit=0,
                             threshold_edit=1).process([13234584]):
        print i
<|endoftext|>


<commit_before>import os
from . import fs_tracker, logs
import uuid
import glob
import time
import filelock

_local_queue_lock = filelock.FileLock(
    os.path.expanduser('~/.studioml/local_queue.lock')
)

class LocalQueue:
    def __init__(self, path=None, verbose=10):
        if path is None:
            self.path = fs_tracker.get_queue_directory()
        else:
            self.path = path
        self.logger = logs.getLogger(self.__class__.__name__)
        self.logger.setLevel(verbose)

    def has_next(self):
        return any(glob.glob(self.path + '/*'))

    def clean(self, timeout=0):
        while self.has_next():
            self.dequeue()

    # Delete and clean are the same for local queue
    def delete(self):
        self.clean()

    def dequeue(self, acknowledge=True, timeout=0):
        wait_step = 1
        for waited in range(0, timeout + wait_step, wait_step):
            with _local_queue_lock:
                files = glob.glob(self.path + '/*')
                if any(files):
                    first_file = min([(p, os.path.getmtime(p)) for p in files],
                                     key=lambda t: t[1])[0]

                    with open(first_file, 'r') as f:
                        data = f.read()

                    self.acknowledge(first_file)
                    if not acknowledge:
                        return data, first_file
                    else:
                        return data

                elif waited == timeout:
                    return None

            # self.logger.info(
            #    ('No messages found, sleeping for {} ' +
            #     ' (total sleep time {})').format(wait_step, waited))
            time.sleep(wait_step)

    def enqueue(self, data):
        with _local_queue_lock:
            filename = os.path.join(self.path, str(uuid.uuid4()))
            with open(filename, 'w') as f:
                f.write(data)

    def acknowledge(self, key):
        try:
            os.remove(key)
        except BaseException:
            pass

    def hold(self, key, minutes):
        self.acknowledge(key)

    def get_name(self):
        return 'local'

    def shutdown(self, delete_queue=True):
        self.delete()

def get_local_queue_lock():
    return _local_queue_lock
<commit_msg>Fixes for LocalQueue status handling. Use specially named file as "queue active" status flag.<commit_after>import os
import sys
from . import fs_tracker, logs
import uuid
import glob
import time
import filelock

_local_queue_lock = filelock.FileLock(
    os.path.expanduser('~/.studioml/local_queue.lock')
)

class LocalQueue:
    def __init__(self, path=None, verbose=10):
        if path is None:
            self.path = fs_tracker.get_queue_directory()
        else:
            self.path = path
        self.logger = logs.getLogger(self.__class__.__name__)
        self.logger.setLevel(verbose)
        self.status_marker = os.path.join(self.path, 'is_active.queue')
        try:
            with open(self.status_marker, "w") as sm:
                pass
        except IOError as error:
            self.logger.error('FAILED to create {0} for LocalQueue. ABORTING.'
                              .format(self.status_marker))
            sys.exit(-1)

    def _get_queue_status(self):
        with _local_queue_lock:
            files = glob.glob(self.path + '/*')
            if files is None:
                files = list()
        is_active = self.status_marker in files
        try:
            files.remove(self.status_marker)
        except:
            # Ignore possible exception:
            # we just want list of files without status marker
            pass

        print(">>>> Status: " + str(is_active) + "  [" + repr(files) + ']')

        return is_active, files

    def has_next(self):
        is_active, files = self._get_queue_status()
        return is_active and len(files) > 0

    def clean(self, timeout=0):
        while self.has_next():
            self.dequeue()

    # Delete and clean are the same for local queue
    def delete(self):
        print("LOCAL QUEUE CLEAN")
        self.clean()
        with _local_queue_lock:
            os.remove(self.status_marker)
        print("LOCAL QUEUE CLEAN DONE")

    def dequeue(self, acknowledge=True, timeout=0):
        wait_step = 1
        for waited in range(0, timeout + wait_step, wait_step):
            with _local_queue_lock:
                is_active, files = self._get_queue_status()
                if not is_active:
                    return None
                if any(files):
                    first_file = min([(p, os.path.getmtime(p)) for p in files],
                                     key=lambda t: t[1])[0]

                    with open(first_file, 'r') as f:
                        data = f.read()

                    self.acknowledge(first_file)
                    if not acknowledge:
                        return data, first_file
                    else:
                        return data

                elif waited == timeout:
                    return None

            # self.logger.info(
            #    ('No messages found, sleeping for {} ' +
            #     ' (total sleep time {})').format(wait_step, waited))
            print('No messages found, sleeping for {0} ({1}) '.format(wait_step, waited))
            time.sleep(wait_step)

    def enqueue(self, data):
        with _local_queue_lock:
            filename = os.path.join(self.path, str(uuid.uuid4()))
            with open(filename, 'w') as f:
                f.write(data)

    def acknowledge(self, key):
        try:
            os.remove(key)
        except BaseException:
            pass

    def hold(self, key, minutes):
        self.acknowledge(key)

    def get_name(self):
        return 'local'

    def shutdown(self, delete_queue=True):
        print("LOCAL QUEUE SHUTDOWN")
        self.delete()
        print("LOCAL QUEUE SHUTDOWN DONE")

def get_local_queue_lock():
    return _local_queue_lock
<|endoftext|>


<commit_before>"""Tools to save/restore model from checkpoints."""

import os

import torch

def load_checkpoint(model, optimizer, model_dir):
    path = os.path.join(model_dir, 'checkpoint')
    if os.path.exists(path):
        print("Loading model from %s" % path)
        checkpoint = torch.load(path)
        old_state_dict = model.state_dict()
        for key in old_state_dict.keys():
            if key not in checkpoint['model']:
                checkpoint['model'][key] = old_state_dict[key]
        model.load_state_dict(checkpoint['model'])
        optimizer.load_state_dict(checkpoint['optimizer'])
        return checkpoint.get('step', 0)
    return 0

def save_checkpoint(model, optimizer, step, model_dir, ignore=[]):
    if not os.path.exists(model_dir):
        os.makedirs(model_dir)
    path = os.path.join(model_dir, 'checkpoint')
    state_dict = model.state_dict()
    if ignore:
        for key in state_dict.keys():
            for item in ignore:
                if key.startswith(item):
                    state_dict.pop(key)
    torch.save({
        'model': state_dict,
        'optimizer': optimizer.state_dict(),
        'step': step
    }, path)

class Saver(object):
    """Class to manage save and restore for the model and optimizer."""

    def __init__(self, model, optimizer):
        self._model = model
        self._optimizer = optimizer

    def restore(self, model_dir):
        """Restores model and optimizer from given directory.

	Returns:
	   Last training step for the model restored.
        """
        last_step = load_checkpoint(self._model, self._optimizer, model_dir)
        return last_step

    def save(self, model_dir, step):
        """Saves model and optimizer to given directory.

        Args:
           model_dir: Model directory to save.
           step: Current training step.
        """
        save_checkpoint(self._model, self._optimizer, step, model_dir)

<commit_msg>fixed import error<commit_after>"""Tools to save/restore model from checkpoints."""

import os
try:
    import torch
except ImportError:
    torch = None

def load_checkpoint(model, optimizer, model_dir):
    path = os.path.join(model_dir, 'checkpoint')
    if os.path.exists(path):
        print("Loading model from %s" % path)
        checkpoint = torch.load(path)
        old_state_dict = model.state_dict()
        for key in old_state_dict.keys():
            if key not in checkpoint['model']:
                checkpoint['model'][key] = old_state_dict[key]
        model.load_state_dict(checkpoint['model'])
        optimizer.load_state_dict(checkpoint['optimizer'])
        return checkpoint.get('step', 0)
    return 0

def save_checkpoint(model, optimizer, step, model_dir, ignore=[]):
    if not os.path.exists(model_dir):
        os.makedirs(model_dir)
    path = os.path.join(model_dir, 'checkpoint')
    state_dict = model.state_dict()
    if ignore:
        for key in state_dict.keys():
            for item in ignore:
                if key.startswith(item):
                    state_dict.pop(key)
    torch.save({
        'model': state_dict,
        'optimizer': optimizer.state_dict(),
        'step': step
    }, path)

class Saver(object):
    """Class to manage save and restore for the model and optimizer."""

    def __init__(self, model, optimizer):
        self._model = model
        self._optimizer = optimizer

    def restore(self, model_dir):
        """Restores model and optimizer from given directory.

	Returns:
	   Last training step for the model restored.
        """
        last_step = load_checkpoint(self._model, self._optimizer, model_dir)
        return last_step

    def save(self, model_dir, step):
        """Saves model and optimizer to given directory.

        Args:
           model_dir: Model directory to save.
           step: Current training step.
        """
        save_checkpoint(self._model, self._optimizer, step, model_dir)

<|endoftext|>