<commit_before>from satoyama import nodetypes
from satoyama.models import *
from uuid import uuid4
from random import random, randint
from numpy.random import uniform

def notest(func):
	setattr(func, 'notest', True)
	return func

class NodeSeeder():

	@staticmethod
	@notest
	def seed_node(node_type_str, **node_args):
		"""
		The omni-node seeder. Use this to create new nodes of any type defined in nodetypes.yml
		:param node_type_str (required): e.g. 'ricefield', 'edge', etc.
		:param latitude (optional): The latitude of the node
		:param longitude (optional): The longitude of the node
		:param alias (optional): The alias of the node
		:param node_readings (optional): The number of dateless dummy data generated per sensor in the node
		"""
		# assert isinstance(node_type_str, str)
		# assert isinstance(latitude, float)
		# assert isinstance(longitude, float)
		# assert isinstance(alias, str)
		# assert isinstance(node_readings, int)

		site = Site.query.filter_by(id = node_args.get('site_id', None)).first()
		latitude = node_args.get('latitude', None)
		longitude = node_args.get('longitude', None)
		alias = node_args.get('alias', uuid4().hex)

		assert node_type_str in nodetypes.keys(), 'Node type "%s" is not defined on this server'%node_type_str
		node_type = NodeType.query.filter(NodeType.name == node_type_str).first()
		if not node_type:
			node_type = NodeType.create(name = node_type_str)

		node = Node.create(node_type = node_type, alias = alias, latitude = latitude, longitude = longitude, site = site)

		for sensor in nodetypes[node_type.name]['sensors']:
			sensortype_unit = sensor['sensortype']['unit']
			sensortype_name = sensor['sensortype']['name']
			sensortype = SensorType.query.filter(SensorType.unit == sensortype_unit and SensorType.name == sensortype_name).first()

			if not sensortype:
				sensortype = SensorType.create(unit = sensortype_unit, name = sensortype_name)

			Sensor.create(sensortype = sensortype, alias = sensor['alias'], node = node)

		n_readings = node_args.get('node_readings', 0)
		for sensor in node.sensors:
			for r in range(n_readings):
				Reading.create(sensor = sensor, value = random())
		return node

		# node_type =

	# @staticmethod
	# @notest
	# def seed_node(node_type, **kwargs):

	# 	assert node_type in satoyama.definitions.node_types
	# 	return getattr(NodeSeeder, 'seed_' + node_type + '_node')(**kwargs)

	@staticmethod
	def seed_ricefield_node(n_readings = 0, **node_args):
		"""
		Create a new ricefield node.
		:param n_readings: (default 0) The number of readings with random values that will be generated for each sensor in the node.
		"""

		if node_args.has_key('site'):
			site = node_args['site']
		elif node_args.has_key('site_id'):
			site = Site.query.filter_by(id = node_args['site_id']).first()
		else:
			site = None

		if node_args.has_key('latitude'):
			latitude = node_args['latitude']
		else:
			latitude = uniform(35.143, 35.144)

		if node_args.has_key('longitude'):
			longitude = node_args['longitude']
		else:
			longitude = uniform(139.988, 139.989)

		if node_args.has_key('alias'):
			alias = node_args['alias']
		else:
			alias = "ricefield_node_%s"%uuid4().hex

		node_type = NodeType.query.filter(NodeType.name == 'ricefield').first()
		if not node_type: node_type = NodeType.create(name = 'ricefield')

		node = Node.create(node_type = node_type, alias = alias, latitude = latitude, longitude = longitude, site = site)

		st_temp = SensorType.query.filter(SensorType.unit == 'C' and SensorType.name == 'temperature').first()
		if not st_temp: st_temp = SensorType.create(unit = 'C', name = 'temperature')
		st_dist = SensorType.query.filter(SensorType.unit == 'cm' and SensorType.name == 'distance').first()
		if not st_dist: st_dist = SensorType.create(unit = 'cm', name = 'distance')
		st_humidity = SensorType.query.filter(SensorType.unit == '%' and SensorType.name == 'humidity').first()
		if not st_humidity: st_humidity = SensorType.create(unit = '%', name = 'humidity')
		st_voltage = SensorType.query.filter(SensorType.unit == 'V' and SensorType.name == 'voltage').first()
		if not st_voltage: st_voltage = SensorType.create(unit = 'V', name = 'voltage')

		Sensor.create(sensortype = st_temp, node = node, alias = 'temperature')
		Sensor.create(sensortype = st_dist, node = node, alias = 'distance')
		Sensor.create(sensortype = st_humidity, node = node, alias = 'humidity')
		Sensor.create(sensortype = st_voltage, node = node, alias = 'vbat')

		for sensor in node.sensors:
			for r in range(n_readings):
				Reading.create(sensor = sensor, value = random())
		return node

<commit_msg>Adds support for populate option. Inserts a reading for each 30 previous days in each sensor<commit_after>from satoyama import nodetypes
from satoyama.models import *
from uuid import uuid4
from random import random, randint
from numpy.random import uniform
from datetime import datetime, timedelta
def notest(func):
	setattr(func, 'notest', True)
	return func

class NodeSeeder():

	@staticmethod
	@notest
	def seed_node(node_type_str, **node_args):
		"""
		The omni-node seeder. Use this to create new nodes of any type defined in nodetypes.yml
		:param node_type_str (required): e.g. 'ricefield', 'edge', etc.
		:param latitude (optional): The latitude of the node
		:param longitude (optional): The longitude of the node
		:param alias (optional): The alias of the node
		:param node_readings (optional): The number of dateless dummy data generated per sensor in the node
		"""
		assert node_type_str in nodetypes.keys(), 'Node type "%s" is not defined on this server'%node_type_str
		# assert isinstance(latitude, float)
		# assert isinstance(longitude, float)
		# assert isinstance(alias, str)
		# assert isinstance(node_readings, int)

		site = Site.query.filter_by(id = node_args.get('site_id', None)).first()
		latitude = node_args.get('latitude', None)
		longitude = node_args.get('longitude', None)
		alias = node_args.get('alias', uuid4().hex)
		populate = node_args.get('populate', None)

		node_type = NodeType.query.filter(NodeType.name == node_type_str).first()
		if not node_type:
			node_type = NodeType.create(name = node_type_str)

		node = Node.create(node_type = node_type, alias = alias, latitude = latitude, longitude = longitude, site = site)

		for sensor in nodetypes[node_type.name]['sensors']:
			sensortype_unit = sensor['sensortype']['unit']
			sensortype_name = sensor['sensortype']['name']
			sensortype = SensorType.query.filter(SensorType.unit == sensortype_unit and SensorType.name == sensortype_name).first()

			if not sensortype:
				sensortype = SensorType.create(unit = sensortype_unit, name = sensortype_name)

			Sensor.create(sensortype = sensortype, alias = sensor['alias'], node = node)

		if populate:
			for sensor in node.sensors:
				for td in range(-30, 0):
					timestamp = datetime.now() + timedelta(td)
					Reading.create(sensor = sensor, value = random(), timestamp = timestamp)
		return node

		# node_type =

	# @staticmethod
	# @notest
	# def seed_node(node_type, **kwargs):

	# 	assert node_type in satoyama.definitions.node_types
	# 	return getattr(NodeSeeder, 'seed_' + node_type + '_node')(**kwargs)

	@staticmethod
	def seed_ricefield_node(n_readings = 0, **node_args):
		"""
		Create a new ricefield node.
		:param n_readings: (default 0) The number of readings with random values that will be generated for each sensor in the node.
		"""

		if node_args.has_key('site'):
			site = node_args['site']
		elif node_args.has_key('site_id'):
			site = Site.query.filter_by(id = node_args['site_id']).first()
		else:
			site = None

		if node_args.has_key('latitude'):
			latitude = node_args['latitude']
		else:
			latitude = uniform(35.143, 35.144)

		if node_args.has_key('longitude'):
			longitude = node_args['longitude']
		else:
			longitude = uniform(139.988, 139.989)

		if node_args.has_key('alias'):
			alias = node_args['alias']
		else:
			alias = "ricefield_node_%s"%uuid4().hex

		node_type = NodeType.query.filter(NodeType.name == 'ricefield').first()
		if not node_type: node_type = NodeType.create(name = 'ricefield')

		node = Node.create(node_type = node_type, alias = alias, latitude = latitude, longitude = longitude, site = site)

		st_temp = SensorType.query.filter(SensorType.unit == 'C' and SensorType.name == 'temperature').first()
		if not st_temp: st_temp = SensorType.create(unit = 'C', name = 'temperature')
		st_dist = SensorType.query.filter(SensorType.unit == 'cm' and SensorType.name == 'distance').first()
		if not st_dist: st_dist = SensorType.create(unit = 'cm', name = 'distance')
		st_humidity = SensorType.query.filter(SensorType.unit == '%' and SensorType.name == 'humidity').first()
		if not st_humidity: st_humidity = SensorType.create(unit = '%', name = 'humidity')
		st_voltage = SensorType.query.filter(SensorType.unit == 'V' and SensorType.name == 'voltage').first()
		if not st_voltage: st_voltage = SensorType.create(unit = 'V', name = 'voltage')

		Sensor.create(sensortype = st_temp, node = node, alias = 'temperature')
		Sensor.create(sensortype = st_dist, node = node, alias = 'distance')
		Sensor.create(sensortype = st_humidity, node = node, alias = 'humidity')
		Sensor.create(sensortype = st_voltage, node = node, alias = 'vbat')

		for sensor in node.sensors:
			for r in range(n_readings):
				Reading.create(sensor = sensor, value = random())
		return node

<|endoftext|>


<commit_before>import os
import re
import sys
import subprocess
import argparse
from testVDRParser import TestVDRParser
from trainVDRParser import TrainVDRParser
from RCNNObjectExtractor import RCNNObjectExtractor
import aux
from vdrDescription import GenerateDescriptions

class cd:
    """Context manager for changing the current working directory"""
    """http://stackoverflow.com/questions/431684/how-do-i-cd-in-python"""
    def __init__(self, newPath):
        self.newPath = newPath

    def __enter__(self):
        self.savedPath = os.getcwd()
        os.chdir(self.newPath)

    def __exit__(self, etype, value, traceback):
        os.chdir(self.savedPath)

class Test:

  def __init__(self, args):
    self.args = args

  def test(self):
    self.extractObjects()
    self.predictVDR()
    self.generateDescriptions()

  '''
  Extract the N-most confident objects from images. These objects represent
  the image for the sake of the VDR Parsing and Image Description.
  '''
  def extractObjects(self):

    print "============================================"
    print "Step 1: Extracting Top-%d objects from images" % self.args.numobjects
    print "============================================"

    # Read the list of files into memory
    target = "test" if self.args.test else "dev"
    targetfiles = open(self.args.images+"/"+target).readlines()
    targetfiles = [x.replace("\n","") for x in targetfiles]

    # General arguments for the RCNNExtractor
    rcnn_args = argparse.Namespace()
    rcnn_args.clustersfile = "%s/objectClusters" % self.args.descriptions
    rcnn_args.hdffile = self.args.images
    rcnn_args.output = self.args.images
    rcnn_args.training = False
    rcnn_args.verbose = self.args.verbose
    rcnn_args.n = self.args.numobjects
    rcnn_args.output = self.args.images

    useful_detections = []

    for f in targetfiles:
      detector = RCNNObjectExtractor(rcnn_args)
      detector.args.imagefilename = "%s/%s" % (self.args.images, re.sub("jpg","hdf", f))
      detector_output = detector.process_hdf()
      conll_file = self.__xml2conll__(rcnn_args.output, f)
      useful_detections.append(conll_file)

    # Now we create target-parsed-train-semi from the examples we managed to find in the data
    handle = open("test_list","w")
    handle.write("cat %s > %s/target-parsed-dev" % (aux.glist_to_str(useful_detections), self.args.images))
    handle.close()
    subprocess.check_call(["sh", "test_list"])
    os.remove("test_list")

  '''
  Use the semi-supervised training data to train a VDR Parsing model.
  '''
  def predictVDR(self):
    print "==============================="
    print "Step 2: Predicting VDR"
    print "==============================="

    if not os.path.isfile("%s/%s" % (self.args.images, "trained.model")):
      self.trainParser()

    pargs = argparse.Namespace()
    pargs.path = self.args.images
    pargs.split = "true"
    pargs.model = "mst"
    pargs.k = 5
    pargs.decoder = "non-proj"
    pargs.semi = "true"
    pargs.runString = "RCNNSemi"
    pargs.useImageFeats = "false"
    pargs.verbose = "false"
    pargs.test = "false"
    vdrParser = TestVDRParser(pargs)
    vdrParser.testVDRParser()

  '''
  Use the semi-supervised training data to train a VDR Parsing model.
  '''
  def trainParser(self, k=5):
    print "==============================="
    print "Step 0: Training the VDR Parser"
    print "==============================="
    pargs = argparse.Namespace()
    pargs.path = self.args.images
    pargs.split = "true"
    pargs.model = "mst"
    pargs.decoder = "non-proj"
    pargs.runString = "RCNNSemi"
    pargs.verbose = "false"
    pargs.semi = "true"
    pargs.k = k
    pargs.useImageFeats = "false"
    vdrParser = TrainVDRParser(pargs)
    vdrParser.trainParser()

  def generateDescriptions(self):
    print "====================================="
    print "Step 3: Generating Image Descriptions"
    print "====================================="

    self.__preparemultibleu__(self.args.descriptions, self.args.test)

    gargs = argparse.Namespace()
    gargs.images = "../"+self.args.images
    gargs.descriptions = "../"+self.args.descriptions
    gargs.model = "4"
    gargs.inducedVDR = "true"
    gargs.first = True
    gargs.rawdata = "../"+self.args.images
    gargs.clusterfile = "../%s/objectClusters" % self.args.images
    gargs.test = False
    gargs.runString = "auto"
    gargs.dictionaries = '/export/scratch2/elliott/language_pickles/'
    gargs.second = False
    gargs.semi = True
    gargs.verbose = self.args.verbose

    with cd("vdrDescription"):
      g = GenerateDescriptions.RunExperiment(gargs)
      g.run_experiment()

  '''
  Reads the content of the XML file into memory and creates a semi-supervised
  VDR from the data. Attaches the person to the object, given the automatically
  calculated spatial relationship.  Any other objects are root attached.
  '''
  def __xml2conll__(self, output_directory, filename):
    semi_objects = aux.parse_xml("%s/%s" % (output_directory, re.sub(r".jpg", ".semi.xml", filename)))
    handle = open("%s/%s" % (output_directory, re.sub(r".jpg", ".jpg.parsed", filename)), "w")

    # We will always write idx+1 because CoNLL-X format is 1-indexed, not 0-indexed
    for idx,o in enumerate(semi_objects):
      oc = aux.centroid(o[1:4])
      o_centroid = "%s|%s|%s" % (oc[0], oc[1], o[5])
      s = str.format("%d\t%s\t%s\tNN\tNN\t%s\t%s\t%s\t_\t_\n" % (idx+1, o[0], o[0], o_centroid, 0, "-"))
      handle.write(s)

    handle.write("\n")
    handle.close()
    if self.args.verbose:
      print "Written semi-supervised CoNLL-X file to %s/%s\n" % (output_directory, re.sub(r".jpg", ".jpg.parsed", filename))
    return "%s/%s" % (output_directory, re.sub(r".jpg", ".jpg.parsed", filename))

  '''
  Prepares the gold-standard image descriptions into files that multi-bleu.perl
  can read.
  '''
  def __preparemultibleu__(self, descPath, testData, n=3):
    # Read the list of files into memory
    target = "test" if testData else "dev"
    targetfiles = open(descPath+"/"+target).readlines()
    targetfiles = [x.replace("\n","") for x in targetfiles]
    targetfiles = [x.replace(".jpg","") for x in targetfiles]

    allfiles = os.listdir(descPath)
    allfiles = sorted([x.replace("\n","") for x in allfiles])

    for i in range(1,n+1):

      clippedfiles = []

      for t in targetfiles:
        for f in allfiles:
          if f.startswith(t):
            if f.endswith("%d.desc" % i):
              clippedfiles.append(f)

      output = open("%s/source-strings-%s-%d-%s" % (descPath, target, i, "first"), "w")

      for x in clippedfiles:
        data = open("%s/%s" % (descPath, x)).readline()
        data = data.split(".")[0]
        output.write(data+".\n")
      output.close()

if __name__ == "__main__":

    parser = argparse.ArgumentParser(description = "Predict descriptions using the self-trained VDR parsing model.")
    parser.add_argument("--images", help="path to the images. Expects one .JPG and one .HDF file per image ")
    parser.add_argument("--descriptions", help="path to the descriptions. Expects X-{1-N} files per image")
    parser.add_argument("--verbose", help="Do you want verbose output?", action="store_true")
    parser.add_argument("--test", help="Run on the test data? Default is dev data", action="store_true")
    parser.add_argument("--numobjects", help="Number of objects to extract. Default=5", default=5, type=int)

    if len(sys.argv)==1:
      parser.print_help()
      sys.exit(1)

    test = Test(parser.parse_args())
    test.test()
<commit_msg>Creating annotation files in separate function; using visFeats in parser; moving comments inside function blocks<commit_after>import os
import re
import sys
import subprocess
import argparse

from testVDRParser import TestVDRParser
from trainVDRParser import TrainVDRParser
from RCNNObjectExtractor import RCNNObjectExtractor
import aux
from vdrDescription import GenerateDescriptions

class cd:
    """Context manager for changing the current working directory"""
    """http://stackoverflow.com/questions/431684/how-do-i-cd-in-python"""
    def __init__(self, newPath):
        self.newPath = newPath

    def __enter__(self):
        self.savedPath = os.getcwd()
        os.chdir(self.newPath)

    def __exit__(self, etype, value, traceback):
        os.chdir(self.savedPath)

class Test:

  def __init__(self, args):
    self.args = args

  def test(self):
    self.extractObjects()
    self.predictVDR()
    self.generateDescriptions()

  def extractObjects(self):
    '''
    Extract the N-most confident objects from images. These objects represent
    the image for the sake of the VDR Parsing and Image Description.
    '''

    print "============================================"
    print "Step 1: Extracting Top-%d objects from images" % self.args.numobjects
    print "============================================"

    # Read the list of files into memory
    target = "test" if self.args.test else "dev"
    targetfiles = open(self.args.images+"/"+target).readlines()
    targetfiles = [x.replace("\n","") for x in targetfiles]

    # General arguments for the RCNNExtractor
    rcnn_args = argparse.Namespace()
    rcnn_args.clustersfile = "%s/objectClusters" % self.args.descriptions
    rcnn_args.hdffile = self.args.images
    rcnn_args.output = self.args.images
    rcnn_args.training = False
    rcnn_args.verbose = self.args.verbose
    rcnn_args.n = self.args.numobjects
    rcnn_args.output = self.args.images

    detections = []

    for f in targetfiles:
      detector = RCNNObjectExtractor(rcnn_args)
      detector.args.imagefilename = "%s/%s" % (self.args.images, re.sub("jpg","hdf", f))
      detector_output = detector.process_hdf()
      conll_file = self.__xml2conll__(rcnn_args.output, f)
      detections.append(conll_file)

    # Now we create target-parsed-train-semi from the examples we managed to find in the data
    handle = open("%s_list" % target,"w")
    handle.write("cat %s > %s/target-parsed-%s" % (aux.glist_to_str(detections), self.args.images, target))
    handle.close()
    subprocess.check_call(["sh", "%s_list" % target])
    os.remove("%s_list" % target)

    self.createAnnotationsFiles(detections)

  def createAnnotationsFiles(self, detections):
    '''
    The annotations file is read by the VDR Parser to get at the object regions directly.
    '''
    target = "test" if self.args.test else "dev"
    ahandle = open("%s/annotations-%s" % (self.args.images, target), "w")
    ihandle = open("%s/images-%s" % (self.args.images, target), "w")
    for f in detections:
      xmlname = re.sub(r".jpg.parsed", ".semi.xml", f)
      jpgname = re.sub(r".jpg.parsed", ".jpg", f)
      prefix = "/export/scratch2/elliott/src/private/"
      ahandle.write("%s/%s\n" % (prefix, xmlname))
      ihandle.write("%s/%s\n" % (prefix, jpgname))

    ahandle.close()
    ihandle.close()

  def predictVDR(self):
    '''
    Use the semi-supervised training data to train a VDR Parsing model.
    '''
    print "==============================="
    print "Step 2: Predicting VDR"
    print "==============================="

    if not os.path.isfile("%s/%s" % (self.args.images, "trained.model")):
      self.trainParser()

    pargs = argparse.Namespace()
    pargs.path = self.args.images
    pargs.split = "true"
    pargs.model = "mst"
    pargs.k = 5
    pargs.decoder = "non-proj"
    pargs.semi = "true"
    pargs.runString = "RCNNSemi"
    pargs.useImageFeats = "true"
    pargs.verbose = "false"
    pargs.test = "true" if self.args.test else "false"
    vdrParser = TestVDRParser(pargs)
    vdrParser.testVDRParser()

  def trainParser(self, k=5):
    '''
    Use the semi-supervised training data to train a VDR Parsing model.
    '''
    print "==============================="
    print "Step 0: Training the VDR Parser"
    print "==============================="
    pargs = argparse.Namespace()
    pargs.path = self.args.images
    pargs.split = "true"
    pargs.model = "mst"
    pargs.decoder = "non-proj"
    pargs.runString = "RCNNSemi"
    pargs.verbose = "false"
    pargs.semi = "true"
    pargs.k = k
    pargs.useImageFeats = "false"
    vdrParser = TrainVDRParser(pargs)
    vdrParser.trainParser()

  def generateDescriptions(self):
    print "====================================="
    print "Step 3: Generating Image Descriptions"
    print "====================================="

    n = 3 if self.args.vlt else 5

    self.__preparemultibleu__(self.args.descriptions, self.args.test, n)

    gargs = argparse.Namespace()
    gargs.images = "../"+self.args.images
    gargs.descriptions = "../"+self.args.descriptions
    gargs.model = "4"
    gargs.inducedVDR = "true"
    gargs.first = True
    gargs.rawdata = "../"+self.args.images
    gargs.clusterfile = "../%s/objectClusters" % self.args.images
    gargs.test = self.args.test
    gargs.runString = "auto"
    gargs.dictionaries = '/export/scratch2/elliott/language_pickles/'
    gargs.second = False
    gargs.semi = True
    gargs.verbose = self.args.verbose
    gargs.gigaword = False
    gargs.postfix = ""

    with cd("vdrDescription"):
      g = GenerateDescriptions.GenerateDescriptions(gargs)
      g.generate()

  def __xml2conll__(self, output_directory, filename):
    '''
    Reads the content of the XML file into memory and creates a semi-supervised
    VDR from the data. Attaches the person to the object, given the automatically
    calculated spatial relationship.  Any other objects are root attached.
    '''
    semi_objects = aux.parse_xml("%s/%s" % (output_directory, re.sub(r".jpg", ".semi.xml", filename)))
    handle = open("%s/%s" % (output_directory, re.sub(r".jpg", ".jpg.parsed", filename)), "w")

    # We will always write idx+1 because CoNLL-X format is 1-indexed, not 0-indexed
    for idx,o in enumerate(semi_objects):
      oc = aux.centroid(o[1:4])
      o_centroid = "%s|%s|%s" % (oc[0], oc[1], o[5])
      s = str.format("%d\t%s\t%s\tNN\tNN\t%s\t%s\t%s\t_\t_\n" % (idx+1, o[0], o[0], o_centroid, 0, "-"))
      handle.write(s)

    handle.write("\n")
    handle.close()
    if self.args.verbose:
      print "Written semi-supervised CoNLL-X file to %s/%s\n" % (output_directory, re.sub(r".jpg", ".jpg.parsed", filename))
    return "%s/%s" % (output_directory, re.sub(r".jpg", ".jpg.parsed", filename))

  def __preparemultibleu__(self, descPath, testData, n=3):
    '''
    Prepares the gold-standard image descriptions into files that multi-bleu.perl
    can read.
    '''
    # Read the list of files into memory
    target = "test" if testData else "dev"
    targetfiles = open(descPath+"/"+target).readlines()
    targetfiles = [x.replace("\n","") for x in targetfiles]
    targetfiles = [x.replace(".jpg","") for x in targetfiles]

    allfiles = os.listdir(descPath)
    allfiles = sorted([x.replace("\n","") for x in allfiles])

    for i in range(1,n+1):

      clippedfiles = []

      for t in targetfiles:
        for f in allfiles:
          if f.startswith(t):
            if f.endswith("%d.desc" % i):
              clippedfiles.append(f)

      output = open("%s/source-strings-%s-%d-%s" % (descPath, target, i, "first"), "w")

      for x in clippedfiles:
        data = open("%s/%s" % (descPath, x)).readline()
        data = data.split(".")[0]
        output.write(data+".\n")
      output.close()

if __name__ == "__main__":

    parser = argparse.ArgumentParser(description = "Predict descriptions using the self-trained VDR parsing model.")
    parser.add_argument("--images", help="path to the images. Expects one .JPG and one .HDF file per image ")
    parser.add_argument("--descriptions", help="path to the descriptions. Expects X-{1-N} files per image")
    parser.add_argument("--verbose", help="Do you want verbose output?", action="store_true")
    parser.add_argument("--test", help="Run on the test data? Default is dev data", action="store_true")
    parser.add_argument("--numobjects", help="Number of objects to extract. Default=5", default=5, type=int)
    parser.add_argument("--vlt", help="VLT data set?", action="store_true")

    if len(sys.argv)==1:
      parser.print_help()
      sys.exit(1)

    test = Test(parser.parse_args())
    test.test()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""
----------------------------------------
Created on Thu Sep 22 15:08:36 2016

@author: Adam Ek
----------------------------------------

Random indexing:

    (1): Generate index vectors
        gather all the words in the data:
            for each word, generate a d-dimensional matrix of 0's with a small amount of (+1) and (-1) distributed

    (2):
        scan text
            each time a word w occurs in a context, add that contexts index vector to the vector of that word

    ######
        UPDATING DATA

"""
import numpy as np
import re
import random
import math
import sklearn.metrics.pairwise as d
import matplotlib.pyplot as plt

class distrib_semantics():

    def __init__(self):
        self.vocabulary = []
        self.i_vectors = []
        self.word_vectors = []
        self.superlist = []

        self.documents = 0
        self.weights = []
        self.total_words = 0
        self.sentences_total = 0

        self.gen_sentences = 0
        self.evaled_data = []

    #create an index vector for each word
    def rand_index_vector(self):
        arr = np.zeros(1024)

        for i in range(0, 4):
            if i%2 == 0:
                arr[random.randint(0, 1023)] = 1
            else:
                arr[random.randint(0, 1023)] = -1

        return arr

    def vectorizer(self, formatted_sentence):
        added = []
        self.sentences_total += 1
        for i, word in enumerate(formatted_sentence):

            #remove special things inside words
            formatted_sentence[i] = re.sub('[^a-zåäö0-9%-]', '', formatted_sentence[i])
            word = re.sub('[^a-zåäö0-9%-]', '', word)

            #len(word) == 0:
            #dont add null words
            if word == '':
                continue #continue does not add ''

            self.total_words += 1
            ###### FINE TUNE DATA
            #change numbers to NUM
            if word.lstrip('-').isdigit():
                word = 'NUM'
                formatted_sentence[i] = 'NUM'
            ### NUMS -> '5-6' etc
            elif re.match('\d+-\d+', word) is not None:
                word = 'NUMS'
                formatted_sentence[i] = 'NUMS'
            #percentages -> 12% etc
            elif '%' in word:
                word = 'PERC'
                formatted_sentence[i] = 'PERC'

            #set up vectors
            if word not in self.vocabulary:
                self.vocabulary.append(word)
                self.i_vectors.append(self.rand_index_vector())
                self.word_vectors.append(np.zeros(1024))
                self.weights.append([0, 1, 1, self.documents])
            #create weight tools
            else:
                self.weights[self.vocabulary.index(word)][1] += 1

                if int(self.documents) in self.weights[self.vocabulary.index(word)][3:]:
                    pass
                else:
                    self.weights[self.vocabulary.index(word)].append(int(self.documents))

                if word not in added:
                    self.weights[self.vocabulary.index(word)][2] += 1
                    added.append(word)

    def create_vectors(self, filenames):
        err = []
        for t, filename in enumerate(filenames):
            try:
                with open(filename) as training:
                    print("Reading file", filename, "...")
                    self.documents += 1
                    for row in training:
                         #separate row into sentences
                         row = row.strip().split('. ')
                         for sentence in row:
                            sentence = sentence.split()
                            #format sentence
                            #remove special things before/after word
                            formatted_sentence = [w.strip().lower() for w in sentence]
                            #only add sentences longer than 0 words
                            if len(formatted_sentence) > 0:
                                #create index_vector for word
                                #currently takes a senttence, change so it takes a word???
                                self.vectorizer(formatted_sentence)

                                #remove [''] and '' elements
                                while '' in formatted_sentence:
                                    formatted_sentence.remove('')

                                self.superlist.append(formatted_sentence)
                    print("Success!\n")

            except FileNotFoundError as fnfe:
                print('FILE ERROR!\n {0}\n'.format(fnfe))
                err.append(filename)
                continue

        #generator for sentences
        self.gen_sentences = (x for x in self.superlist)

        #return values
        if err:
            return err
        else:
            return None

    #create weights and add index vectors to word vectors
    def apply(self):
        self.apply_weights()
        self.create_word_vectors()

    ######## weights always updated after create_vectors, incorporate
    # n-gram history? could be saved

    #Assign weight values
    def apply_weights(self):
        print("Applying weights...\n")
        ###### Differrent weights if word is behind/after???
        ##Create weights for indexes
        for i, w in enumerate(self.weights):
#            self.word_vectors[i] = np.zeros(1024) #updating
            #real idf
            idf = math.log1p(self.documents/len(self.weights[i][3:])) #smooth
            isf = math.log(self.sentences_total/w[2]) #inverse sentence frequency
            self.weights[i][0] = (w[1]/self.total_words)*idf

    #add index vectors in the words context to the word_vector
    def create_word_vectors(self):
        print("Creating word vectors...\n")
        window = 1 #how many words before/after to consider being a part of the context
        #weight for sliding window > 1
        #create contexts
        for sentence in self.gen_sentences:
            for i, word in enumerate(sentence):
                ###SLIDING WINDOW PARAMETER
                for n in range(1,window+1):
                    #words before
                    try:
                        if i - n >= 0: #exclude negative indexes
                            prev_word = self.vocabulary.index(sentence[i-n])
                            self.word_vectors[self.vocabulary.index(word)] += (self.i_vectors[prev_word] * self.weights[prev_word][0])
                    except:
                        pass

                    #words after
                    try:
                        next_word = self.vocabulary.index(sentence[i+n])
                        self.word_vectors[self.vocabulary.index(word)] += (self.i_vectors[next_word] * self.weights[next_word][0])
                    except:
                        pass
        #empty sentence list incase update command
#        self.superlist = []

    #######################################################
    ################## EVUALUATIONS #######################
    #######################################################
    def find_similarity(self, word1, word2):

        #check if the words exists
        if word1 not in self.vocabulary:
            return ' '.join([word1, "does not exist, try again"])
        elif word2 not in self.vocabulary:
            return ' '.join([word2, "does not exist, try again"])
        else:
            i_word1 = self.word_vectors[self.vocabulary.index(word1)]
            i_word2 = self.word_vectors[self.vocabulary.index(word2)]

        ### Check definition and implementation
        cosine_sim = d.cosine_similarity(i_word1.reshape(1,-1), i_word2.reshape(1,-1))

        return(cosine_sim[0][0])

    #top 3
    def similarity_top(self, word):

        if word not in self.vocabulary:
            return ' '.join([word, "does not exist, try again"])
        else:
            ind = self.vocabulary.index(word)
            word = self.word_vectors[ind]

        top = [0,0,0]
        word_top = ["","",""]

        for i, vect in enumerate(self.word_vectors):
            if i == ind:
                continue

            cs = d.cosine_similarity(word.reshape(1,-1), vect.reshape(1,-1))

            if cs > top[0]:
                top[0] = cs
                word_top[0] = self.vocabulary[i]
            elif cs > top[1]:
                top[1] = cs
                word_top[1] = self.vocabulary[i]
            elif cs > top[2]:
                top[2] = cs
                word_top[2] = self.vocabulary[i]

        return [(x, y[0][0]) for x, y in zip(word_top,top)]

    def graph(self):

        points = []

        for val in self.weights:
            points.append(val[1])

        plt.scatter(points, points)
        plt.show()

    #######################################################
    ################## DATA OPTIONS #######################
    #######################################################
    #save data
    def save(self):
        ofile = '/home/usr1/git/dist_data/np_data.npz'
        np.savez(ofile,
                 v = self.vocabulary,
                 wv = self.word_vectors,
                 iv = self.i_vectors,
                 w = self.weights,
                 tw = self.total_words,
                 st = self.sentences_total,
                 d = self.documents)

    #load saved data
    def load(self):
        data = np.load('/home/usr1/git/dist_data/np_data.npz')

        self.vocabulary = list(data['v'])
        self.i_vectors = list(data['iv'])
        self.word_vectors = list(data['wv'])
        self.weights = list(data['w'])
        self.total_words = int(data['tw'])
        self.sentences_total = int(data['st'])
        self.documents = int(data['d'])

    #update the data
    def update(self, paths):
        #read new files
        for path in paths:
            path.rstrip(',')
            z = self.create_vectors([path])

            ### HANDLE ERRORS
        if z != None:
            print("Error reading", z)
        else:
            #apply the new data
            self.apply()
            print("New data successfully applied")

    #info about the data or individual words
    def info(self, arg):

        if arg != None:
            if arg not in self.vocabulary:
                print(arg, 'does not exist')
            else:
                print('occurences:', self.weights[self.vocabulary.index(arg)][1])
                print('in', self.weights[self.vocabulary.index(arg)][2], 'sentences')
                print('word in documents:', self.weights[self.vocabulary.index(arg)][3:])
                print('tf:', self.weights[self.vocabulary.index(arg)][1]/self.total_words)
                print('idf:', math.log1p(self.documents/len(self.weights[self.vocabulary.index(arg)][3:])))
                print('total weight of word:', self.weights[self.vocabulary.index(arg)][0],'\n')
        else:
            print(len(self.vocabulary), 'unique words in vocabulary')
            print(self.sentences_total, 'total sentences')
            print(self.total_words, 'total words')
            print(self.documents, 'total documents\n')

#######################################################
##################### INTERFACE #######################
#######################################################
def main():

    x = distrib_semantics()

    print("Welcome to Distributial Semantics with Random Indexing")
    new_data = False
    #load data
    while True:
        if new_data == True:
            print("Enter paths by typing 'new <path>' finish by typing 'apply'")
        else:
            print("Enter new data source by typing 'new <path>' and load saved data by typing 'load'")

        setup = input('> ')
        setup = setup.split()

        if setup[0] == 'load':
            if new_data == False:
                x.load()
                break

        elif setup[0] == 'new':
            new_data = True
            x.create_vectors(['/home/usr1/PythonPrg/project/test_doc_1.txt','/home/usr1/PythonPrg/pr111oject/test_doc_2.txt', '/home/usr1/PythonPrg/project/test_doc_3.txt', '/home/usr1/PythonPrg/project/test_doc_4.txt'])

        elif setup[0] == 'apply':
            if new_data == True:
                x.apply()
                break

        else:
            print('Invalid input')

    #User interface after data has loaded
    print("Type 'sim <word1> <word2>' for similarity between two words, 'top <word>' for top 3 similar words, 'help' to display availible commands and 'exit' to quit")
    while True:
        choice = input('> ')
        input_args = choice.split()

        if len(input_args) == 0:
            print('Please try again')

        #fix error output
        elif input_args[0] == 'sim':
            try:
                sim_res = x.find_similarity(input_args[1].lower(), input_args[2].lower())
                if sim_res == str(sim_res):
                    print(sim_res)
                else:
                    print('Cosine similarity between', input_args[1], 'and',input_args[2] ,'is\n', sim_res)
            except:
                print("Invalid input for 'sim'")

        #DO NOT USE, CONTAINS ERRORS
        #fix error output
        elif input_args[0] == 'top':
            try:
                top_res = x.similarity_top(input_args[1].lower())
                if top_res == str(top_res):
                    print(top_res)
                else:
                    print('Top similar words for', input_args[1], 'is:')
                    for i, (x, y) in enumerate(top_res):
                        print(i+1, x, y)
            except:
                print("Invalid input for 'top'")

        elif input_args[0] == 'exit':
           break

        elif input_args[0] == 'save':
            x.save()

        elif input_args[0] == 'update':
            try:
                x.update(input_args[1:])
            except:
                print('Please provide a path/paths')

        elif input_args[0] == 'info':
            try:
                x.info(input_args[1].lower())
            except:
                x.info(None)

        elif input_args[0] == 'graph':
            x.graph()

        elif input_args[0] == 'help':
            print("- Semantic operations")
            print("'sim <word1> <word2>' for similarity")
            print("'top <word>' for top 3 similar words")
            print("- Data operations")
            print("'save' to save current data")
            print("'update <path>' to update the data with a new textfile")
            print("'info' for info about the data")
            print("'info <word> for info about the word'")
            print("- ETC")
            print("'exit' to quit")

        else:
           print("Unrecognized command")

if __name__ == '__main__':
    main()

<commit_msg>update<commit_after># -*- coding: utf-8 -*-
"""
----------------------------------------
Created on Thu Sep 22 15:08:36 2016

@author: Adam Ek
----------------------------------------

Random indexing:

    (1): Generate index vectors
        gather all the words in the data:
            for each word, generate a d-dimensional matrix of 0's with a small amount of (+1) and (-1) distributed

    (2):
        scan text
            each time a word w occurs in a context, add that contexts index vector to the vector of that word

"""
import numpy as np
import re
import random
import math
import sklearn.metrics.pairwise as d
import matplotlib.pyplot as plt

class distrib_semantics():

    def __init__(self):
        self.vocabulary = []
        self.i_vectors = []
        self.word_vectors = []
        self.superlist = []

        self.documents = 0
        self.weights = []
        self.total_words = 0
        self.sentences_total = 0

        self.gen_sentences = 0
        self.evaled_data = []

    #create an index vector for each word
    def rand_index_vector(self):
        arr = np.zeros(1024)

        for i in range(0, 4):
            if i%2 == 0:
                arr[random.randint(0, 1023)] = 1
            else:
                arr[random.randint(0, 1023)] = -1

        return arr

    def vectorizer(self, formatted_sentence):
        added = []
        self.sentences_total += 1
        for i, word in enumerate(formatted_sentence):

            #remove special things inside words
            formatted_sentence[i] = re.sub('[^a-zåäö0-9%-]', '', formatted_sentence[i])
            word = re.sub('[^a-zåäö0-9%-]', '', word)

            #len(word) == 0:
            #dont add null words
            if word == '':
                continue #continue does not add ''

            self.total_words += 1
            ###### FINE TUNE DATA
            #change numbers to NUM
            if word.lstrip('-').isdigit():
                word = 'NUM'
                formatted_sentence[i] = 'NUM'
            ### NUMS -> '5-6' etc
            elif re.match('\d+-\d+', word) is not None:
                word = 'NUMS'
                formatted_sentence[i] = 'NUMS'
            #percentages -> 12% etc
            elif '%' in word:
                word = 'PERC'
                formatted_sentence[i] = 'PERC'

            #set up vectors
            if word not in self.vocabulary:
                self.vocabulary.append(word)
                self.i_vectors.append(self.rand_index_vector())
                self.word_vectors.append(np.zeros(1024))
                self.weights.append([0, 1, 1, self.documents])
            #create weight tools
            else:
                self.weights[self.vocabulary.index(word)][1] += 1

                if int(self.documents) in self.weights[self.vocabulary.index(word)][3:]:
                    pass
                else:
                    self.weights[self.vocabulary.index(word)].append(int(self.documents))

                if word not in added:
                    self.weights[self.vocabulary.index(word)][2] += 1
                    added.append(word)

    def create_vectors(self, filenames):
        err = []
        for t, filename in enumerate(filenames):
            try:
                with open(filename) as training:
                    print("Reading file", filename, "...")
                    self.documents += 1
                    for row in training:
                         #separate row into sentences
                         row = row.strip().split('. ')
                         for sentence in row:
                            sentence = sentence.split()
                            #format sentence
                            #remove special things before/after word
                            formatted_sentence = [w.strip().lower() for w in sentence]
                            #only add sentences longer than 0 words
                            if len(formatted_sentence) > 0:
                                #create index_vector for word
                                #currently takes a senttence, change so it takes a word???
                                self.vectorizer(formatted_sentence)

                                #remove [''] and '' elements
                                while '' in formatted_sentence:
                                    formatted_sentence.remove('')

                                self.superlist.append(formatted_sentence)
                    print("Success!\n")

            except FileNotFoundError as fnfe:
                print('FILE ERROR!\n {0}\n'.format(fnfe))
                err.append(filename)
                continue

        #generator for sentences

        #return values
        if err:
            return err
        else:
            return None

    #create weights and add index vectors to word vectors
    def apply(self, update=False):
        self.apply_weights()
        self.create_word_vectors(update)

    ######## weights always updated after create_vectors, incorporate
    # n-gram history? could be saved

    #Assign weight values
    def apply_weights(self):
        print("Applying weights...\n")
        ###### Differrent weights if word is behind/after???
        ##Create weights for indexes
        for i, w in enumerate(self.weights):
            self.word_vectors[i] = np.zeros(1024) #updating
            #real idf
            idf = math.log1p(self.documents/len(self.weights[i][3:])) #smooth
#            isf = math.log(self.sentences_total/w[2]) #inverse sentence frequency
            self.weights[i][0] = (w[1]/self.total_words)*idf

    #default update = False
    #add index vectors in the words context to the word_vector
    def create_word_vectors(self, update):
        print("Creating word vectors...\n")
        self.window = 1 #how many words before/after to consider being a part of the context
        #weight for sliding window > 1
        self.gen_sentences = (x for x in self.superlist)

        for sentence in self.gen_sentences:
                self.ngram_contexts(sentence)

        #create contexts
        if update == True:
            self.upgrade_contexts()

        self.superlist = []

    def ngram_contexts(self, sentence):
        for i, word in enumerate(sentence):
            ###SLIDING WINDOW PARAMETER
            for n in range(1,self.window+1):
                #words before
                try:
                    if i - n >= 0: #exclude negative indexes
                        prev_word = self.vocabulary.index(sentence[i-n])
                        self.word_vectors[self.vocabulary.index(word)] += (self.i_vectors[prev_word] * self.weights[prev_word][0])
                        self.evaled_data.append((self.vocabulary.index(word), prev_word))
                except:
                    pass

                #words after
                try:
                    next_word = self.vocabulary.index(sentence[i+n])
                    self.word_vectors[self.vocabulary.index(word)] += (self.i_vectors[next_word] * self.weights[next_word][0])
                    self.evaled_data.append((self.vocabulary.index(word), next_word))
                except:
                    pass

    def upgrade_contexts(self):
        print("Re-applying updated vectors...\n")
        data = np.load('/home/usr1/git/dist_data/np_hist.npy')

        for x, y in data:
            self.word_vectors[x] += (self.i_vectors[y] * self.weights[y][0])

        #drop data
        data = []

    #######################################################
    ################## EVUALUATIONS #######################
    #######################################################
    def find_similarity(self, word1, word2):

        #check if the words exists
        if word1 not in self.vocabulary:
            return ' '.join([word1, "does not exist, try again"])
        elif word2 not in self.vocabulary:
            return ' '.join([word2, "does not exist, try again"])
        else:
            i_word1 = self.word_vectors[self.vocabulary.index(word1)]
            i_word2 = self.word_vectors[self.vocabulary.index(word2)]

        ### Check definition and implementation
        cosine_sim = d.cosine_similarity(i_word1.reshape(1,-1), i_word2.reshape(1,-1))

        return(cosine_sim[0][0])

    #top 3
    def similarity_top(self, word):

        if word not in self.vocabulary:
            return ' '.join([word, "does not exist, try again"])
        else:
            ind = self.vocabulary.index(word)
            word = self.word_vectors[ind]

        top = [0,0,0]
        word_top = ["","",""]

        for i, vect in enumerate(self.word_vectors):
            if i == ind:
                continue

            cs = d.cosine_similarity(word.reshape(1,-1), vect.reshape(1,-1))

            if cs > top[0]:
                top[0] = cs
                word_top[0] = self.vocabulary[i]
            elif cs > top[1]:
                top[1] = cs
                word_top[1] = self.vocabulary[i]
            elif cs > top[2]:
                top[2] = cs
                word_top[2] = self.vocabulary[i]

        return [(x, y[0][0]) for x, y in zip(word_top,top)]

    def graph(self):

        points = []

        for val in self.weights:
            points.append(val[1])

        plt.scatter(points, points)
        plt.show()

    #######################################################
    ################## DATA OPTIONS #######################
    #######################################################
    #save data
    def save(self):
        ofile = '/home/usr1/git/dist_data/np_data.npz'
        np.savez(ofile,
                 v = self.vocabulary,
                 wv = self.word_vectors,
                 iv = self.i_vectors,
                 w = self.weights,
                 tw = self.total_words,
                 st = self.sentences_total,
                 d = self.documents)

        histfile = '/home/usr1/git/dist_data/np_hist.npy'
        np.save(histfile, self.evaled_data)

    #load saved data
    def load(self):
        data = np.load('/home/usr1/git/dist_data/np_data.npz')

        self.vocabulary = list(data['v'])
        self.i_vectors = list(data['iv'])
        self.word_vectors = list(data['wv'])
        self.weights = list(data['w'])
        self.total_words = int(data['tw'])
        self.sentences_total = int(data['st'])
        self.documents = int(data['d'])

    #update the data
    def update(self, paths):
        #read new files
        for path in paths:
            path.rstrip(',')
            z = self.create_vectors([path])

            ### HANDLE ERRORS
        if z != None:
            print("Error reading", z)
        else:
            #apply the new data
            self.apply(True)
            print("New data successfully applied")

    #info about the data or individual words
    def info(self, arg):

        if arg != None:
            if arg not in self.vocabulary:
                print(arg, 'does not exist')
            else:
                print('occurences:', self.weights[self.vocabulary.index(arg)][1])
                print('in', self.weights[self.vocabulary.index(arg)][2], 'sentences')
                print('word in documents:', self.weights[self.vocabulary.index(arg)][3:])
                print('tf:', self.weights[self.vocabulary.index(arg)][1]/self.total_words)
                print('idf:', math.log1p(self.documents/len(self.weights[self.vocabulary.index(arg)][3:])))
                print('total weight of word:', self.weights[self.vocabulary.index(arg)][0],'\n')
        else:
            print(len(self.vocabulary), 'unique words in vocabulary')
            print(self.sentences_total, 'total sentences')
            print(self.total_words, 'total words')
            print(self.documents, 'total documents\n')

#######################################################
##################### INTERFACE #######################
#######################################################
def main():

    x = distrib_semantics()

    print("Welcome to Distributial Semantics with Random Indexing")
    new_data = False
    #load data
    while True:
        if new_data == True:
            print("Enter paths by typing 'new <path>' finish by typing 'apply'")
        else:
            print("Enter new data source by typing 'new <path>' and load saved data by typing 'load'")

        setup = input('> ')
        setup = setup.split()

        if setup[0] == 'load':
            if new_data == False:
                x.load()
                break

        elif setup[0] == 'new':
            new_data = True
            x.create_vectors(['/home/usr1/PythonPrg/project/test_doc_1.txt','/home/usr1/PythonPrg/pr111oject/test_doc_2.txt', '/home/usr1/PythonPrg/project/test_doc_3.txt', '/home/usr1/PythonPrg/project/test_doc_4.txt'])

        elif setup[0] == 'apply':
            if new_data == True:
                x.apply()
                break

        else:
            print('Invalid input')

    #User interface after data has loaded
    print("Type 'sim <word1> <word2>' for similarity between two words, 'top <word>' for top 3 similar words, 'help' to display availible commands and 'exit' to quit")
    while True:
        choice = input('> ')
        input_args = choice.split()

        if len(input_args) == 0:
            print('Please try again')

        #fix error output
        elif input_args[0] == 'sim':
            try:
                sim_res = x.find_similarity(input_args[1].lower(), input_args[2].lower())
                if sim_res == str(sim_res):
                    print(sim_res)
                else:
                    print('Cosine similarity between', input_args[1], 'and',input_args[2] ,'is\n', sim_res)
            except:
                print("Invalid input for 'sim'")

        #DO NOT USE, CONTAINS ERRORS
        #fix error output
        elif input_args[0] == 'top':
            try:
                top_res = x.similarity_top(input_args[1].lower())
                if top_res == str(top_res):
                    print(top_res)
                else:
                    print('Top similar words for', input_args[1], 'is:')
                    for i, (x, y) in enumerate(top_res):
                        print(i+1, x, y)
            except:
                print("Invalid input for 'top'")

        elif input_args[0] == 'exit':
           break

        elif input_args[0] == 'save':
            x.save()

        elif input_args[0] == 'update':
            try:
                x.update(input_args[1:])
            except:
                print('Please provide a path/paths')

        elif input_args[0] == 'info':
            try:
                x.info(input_args[1].lower())
            except:
                x.info(None)

        elif input_args[0] == 'graph':
            x.graph()

        elif input_args[0] == 'help':
            print("- Semantic operations")
            print("\t'sim <word1> <word2>' for similarity")
            print("\t'top <word>' for top 3 similar words")
            print("- Data operations")
            print("\t'save' to save current data")
            print("\t'update <path>' to update the data with a new textfile")
            print("\t'info' for info about the data")
            print("\t'info <word> for info about the word'")
            print("- ETC")
            print("\t'exit' to quit")

        else:
           print("Unrecognized command")

if __name__ == '__main__':
    main()

<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

__author__ = 'XESS Corporation'
__email__ = 'info@xess.com'

from .local import *

# Place information about this distributor into the distributor dictionary.
from .. import distributors
distributors.update(
    {
        'local_template': {
            'module': 'local', # The directory name containing this file.
            'scrape': 'local', # Allowable values: 'web' or 'local'.
            'label': 'Local',  # Distributor label used in spreadsheet columns.
            'order_cols': ['part_num', 'purch', 'refs'],  # Sort-order for online orders.
            'order_delimiter': ' ',  # Delimiter for online orders.
            # Formatting for distributor header in worksheet.
            'wrk_hdr_format': {
                'font_size': 14,
                'font_color': 'white',
                'bold': True,
                'align': 'center',
                'valign': 'vcenter',
                'bg_color': '#008000'  # Darker green.
            }
        }
    }
)
<commit_msg>Delete __init__.py<commit_after><|endoftext|>


<commit_before># Quick tests for the markup templatetags (django.contrib.markup)

import re
import unittest

from django.template import Template, Context, add_to_builtins
from django.utils.html import escape

add_to_builtins('django.contrib.markup.templatetags.markup')

class Templates(unittest.TestCase):
    def test_textile(self):
        try:
            import textile
        except ImportError:
            textile = None

        textile_content = """Paragraph 1

Paragraph 2 with "quotes" and @code@"""

        t = Template("{{ textile_content|textile }}")
        rendered = t.render(Context(locals())).strip()
        if textile:
            self.assertEqual(rendered, """
Paragraph 1

Paragraph 2 with “quotes” and code
""")
        else:
            self.assertEqual(rendered, escape(textile_content))

    def test_markdown(self):
        try:
            import markdown
        except ImportError:
            markdown = None

        markdown_content = """Paragraph 1

## An h2"""

        t = Template("{{ markdown_content|markdown }}")
        rendered = t.render(Context(locals())).strip()
        if markdown:
            pattern = re.compile("""
Paragraph 1\s*
\s*
\s*An h2
""")
            self.assert_(pattern.match(rendered))
        else:
            self.assertEqual(rendered, markdown_content)

    def test_docutils(self):
        try:
            import docutils
        except ImportError:
            docutils = None

        rest_content = """Paragraph 1

Paragraph 2 with a link_

.. _link: http://www.example.com/"""

        t = Template("{{ rest_content|restructuredtext }}")
        rendered = t.render(Context(locals())).strip()
        if docutils:
            self.assertEqual(rendered, """
Paragraph 1

Paragraph 2 with a link
""")
        else:
            self.assertEqual(rendered, rest_content)

if __name__ == '__main__':
    unittest.main()
<commit_msg>Fixed #5362: markup tests no longer fail under SVN versions of docutils. Thanks, keithb.<commit_after># Quick tests for the markup templatetags (django.contrib.markup)

import re
import unittest

from django.template import Template, Context, add_to_builtins
from django.utils.html import escape

add_to_builtins('django.contrib.markup.templatetags.markup')

class Templates(unittest.TestCase):
    def test_textile(self):
        try:
            import textile
        except ImportError:
            textile = None

        textile_content = """Paragraph 1

Paragraph 2 with "quotes" and @code@"""

        t = Template("{{ textile_content|textile }}")
        rendered = t.render(Context(locals())).strip()
        if textile:
            self.assertEqual(rendered, """
Paragraph 1

Paragraph 2 with “quotes” and code
""")
        else:
            self.assertEqual(rendered, escape(textile_content))

    def test_markdown(self):
        try:
            import markdown
        except ImportError:
            markdown = None

        markdown_content = """Paragraph 1

## An h2"""

        t = Template("{{ markdown_content|markdown }}")
        rendered = t.render(Context(locals())).strip()
        if markdown:
            pattern = re.compile("""
Paragraph 1\s*
\s*
\s*An h2
""")
            self.assert_(pattern.match(rendered))
        else:
            self.assertEqual(rendered, markdown_content)

    def test_docutils(self):
        try:
            import docutils
        except ImportError:
            docutils = None

        rest_content = """Paragraph 1

Paragraph 2 with a link_

.. _link: http://www.example.com/"""

        t = Template("{{ rest_content|restructuredtext }}")
        rendered = t.render(Context(locals())).strip()
        if docutils:
            # Different versions of docutils return slightly different HTML
            try:
                # Docutils v0.4 and earlier
                self.assertEqual(rendered, """
Paragraph 1

Paragraph 2 with a link
""")
            except AssertionError, e:
                # Docutils from SVN (which will become 0.5)
                self.assertEqual(rendered, """
Paragraph 1

Paragraph 2 with a link
""")
        else:
            self.assertEqual(rendered, rest_content)

if __name__ == '__main__':
    unittest.main()
<|endoftext|>


<commit_before>"""
FastCGI server that implements the WSGI protocol.

Uses the flup python package: http://www.saddi.com/software/flup/

This is a adaptation of the flup package to add FastCGI server support
to run Django apps from Web servers that support the FastCGI protocol.
This module can be run standalone or from the django-admin / manage.py
scripts using the "runfcgi" directive.

Run with the extra option "help" for a list of additional options you can
pass to this server.
"""

import sys, os

__version__ = "0.1"
__all__ = ["runfastcgi"]

FASTCGI_HELP = r"""runfcgi:
  Run this project as a fastcgi application. To do this, the
  flup package from http://www.saddi.com/software/flup/ is
  required.

Usage:
   django-admin.py runfcgi --settings=yourproject.settings [fcgi settings]
   manage.py runfcgi [fcgi settings]

Optional Fcgi settings: (setting=value)
  host=HOSTNAME        hostname to listen on..
  port=PORTNUM         port to listen on.
  socket=FILE          UNIX socket to listen on.
  method=IMPL          prefork or threaded (default prefork)
  maxrequests=NUMBER   number of requests a child handles before it is
                       killed and a new child is forked (0 = no limit).
  maxspare=NUMBER      max number of spare processes to keep running.
  minspare=NUMBER      min number of spare processes to prefork.
  maxchildren=NUMBER   hard limit number of processes in prefork mode.
  daemonize=BOOL       whether to detach from terminal.
  pidfile=FILE         write the spawned process-id to this file.
  workdir=DIRECTORY    change to this directory when daemonizing

Examples:
  Run a "standard" fastcgi process on a file-descriptor
  (for webservers which spawn your processes for you)
    $ manage.py runfcgi method=threaded

  Run a fastcgi server on a TCP host/port
    $ manage.py runfcgi method=prefork host=127.0.0.1 port=8025

  Run a fastcgi server on a UNIX domain socket (posix platforms only)
    $ manage.py runfcgi method=prefork socket=/tmp/fcgi.sock

  Run a fastCGI as a daemon and write the spawned PID in a file
    $ manage.py runfcgi socket=/tmp/fcgi.sock method=prefork \
        daemonize=true pidfile=/var/run/django-fcgi.pid

"""

FASTCGI_OPTIONS = {
    'host': None,
    'port': None,
    'socket': None,
    'method': 'fork',
    'daemonize': None,
    'workdir': '/',
    'pidfile': None,
    'maxspare': 5,
    'minspare': 2,
    'maxchildren': 50,
    'maxrequests': 0,
}

def fastcgi_help(message=None):
    print FASTCGI_HELP
    if message:
        print message
    return False

def runfastcgi(argset=[], **kwargs):
    options = FASTCGI_OPTIONS.copy()
    options.update(kwargs)
    for x in argset:
        if "=" in x:
            k, v = x.split('=', 1)
        else:
            k, v = x, True
        options[k.lower()] = v

    if "help" in options:
        return fastcgi_help()

    try:
        import flup
    except ImportError, e:
        print >> sys.stderr, "ERROR: %s" % e
        print >> sys.stderr, "  Unable to load the flup package.  In order to run django"
        print >> sys.stderr, "  as a FastCGI application, you will need to get flup from"
        print >> sys.stderr, "  http://www.saddi.com/software/flup/   If you've already"
        print >> sys.stderr, "  installed flup, then make sure you have it in your PYTHONPATH."
        return False

    if options['method'] in ('prefork', 'fork'):
        from flup.server.fcgi_fork import WSGIServer
        wsgi_opts = {
            'maxSpare': int(options["maxspare"]),
            'minSpare': int(options["minspare"]),
            'maxChildren': int(options["maxchildren"]),
            'maxRequests': int(options["maxrequests"]),
        }
    elif options['method'] in ('thread', 'threaded'):
        from flup.server.fcgi import WSGIServer
        wsgi_opts = {}
    else:
        return fastcgi_help("ERROR: Implementation must be one of prefork or thread.")

    # Prep up and go
    from django.core.handlers.wsgi import WSGIHandler

    if options["host"] and options["port"] and not options["socket"]:
        wsgi_opts['bindAddress'] = (options["host"], int(options["port"]))
    elif options["socket"] and not options["host"] and not options["port"]:
        wsgi_opts['bindAddress'] = options["socket"]
    elif not options["socket"] and not options["host"] and not options["port"]:
        wsgi_opts['bindAddress'] = None
    else:
        return fastcgi_help("Invalid combination of host, port, socket.")

    if options["daemonize"] is None:
        # Default to daemonizing if we're running on a socket/named pipe.
        daemonize = (wsgi_opts['bindAddress'] is not None)
    else:
        if options["daemonize"].lower() in ('true', 'yes', 't'):
            daemonize = True
        elif options["daemonize"].lower() in ('false', 'no', 'f'):
            daemonize = False
        else:
            return fastcgi_help("ERROR: Invalid option for daemonize parameter.")

    if daemonize:
        from django.utils.daemonize import become_daemon
        become_daemon(our_home_dir=options["workdir"])

    if options["pidfile"]:
        fp = open(options["pidfile"], "w")
        fp.write("%d\n" % os.getpid())
        fp.close()

    WSGIServer(WSGIHandler(), **wsgi_opts).run()

if __name__ == '__main__':
    runfastcgi(sys.argv[1:])
<commit_msg>Fixed $2973: added minspare/maxspare/maxchildren options to runfcgi.  Thanks, James Crasta.<commit_after>"""
FastCGI server that implements the WSGI protocol.

Uses the flup python package: http://www.saddi.com/software/flup/

This is a adaptation of the flup package to add FastCGI server support
to run Django apps from Web servers that support the FastCGI protocol.
This module can be run standalone or from the django-admin / manage.py
scripts using the "runfcgi" directive.

Run with the extra option "help" for a list of additional options you can
pass to this server.
"""

import sys, os

__version__ = "0.1"
__all__ = ["runfastcgi"]

FASTCGI_HELP = r"""runfcgi:
  Run this project as a fastcgi application. To do this, the
  flup package from http://www.saddi.com/software/flup/ is
  required.

Usage:
   django-admin.py runfcgi --settings=yourproject.settings [fcgi settings]
   manage.py runfcgi [fcgi settings]

Optional Fcgi settings: (setting=value)
  host=HOSTNAME        hostname to listen on..
  port=PORTNUM         port to listen on.
  socket=FILE          UNIX socket to listen on.
  method=IMPL          prefork or threaded (default prefork)
  maxrequests=NUMBER   number of requests a child handles before it is
                       killed and a new child is forked (0 = no limit).
  maxspare=NUMBER      max number of spare processes / threads
  minspare=NUMBER      min number of spare processes / threads.
  maxchildren=NUMBER   hard limit number of processes / threads
  daemonize=BOOL       whether to detach from terminal.
  pidfile=FILE         write the spawned process-id to this file.
  workdir=DIRECTORY    change to this directory when daemonizing

Examples:
  Run a "standard" fastcgi process on a file-descriptor
  (for webservers which spawn your processes for you)
    $ manage.py runfcgi method=threaded

  Run a fastcgi server on a TCP host/port
    $ manage.py runfcgi method=prefork host=127.0.0.1 port=8025

  Run a fastcgi server on a UNIX domain socket (posix platforms only)
    $ manage.py runfcgi method=prefork socket=/tmp/fcgi.sock

  Run a fastCGI as a daemon and write the spawned PID in a file
    $ manage.py runfcgi socket=/tmp/fcgi.sock method=prefork \
        daemonize=true pidfile=/var/run/django-fcgi.pid

"""

FASTCGI_OPTIONS = {
    'host': None,
    'port': None,
    'socket': None,
    'method': 'fork',
    'daemonize': None,
    'workdir': '/',
    'pidfile': None,
    'maxspare': 5,
    'minspare': 2,
    'maxchildren': 50,
    'maxrequests': 0,
}

def fastcgi_help(message=None):
    print FASTCGI_HELP
    if message:
        print message
    return False

def runfastcgi(argset=[], **kwargs):
    options = FASTCGI_OPTIONS.copy()
    options.update(kwargs)
    for x in argset:
        if "=" in x:
            k, v = x.split('=', 1)
        else:
            k, v = x, True
        options[k.lower()] = v

    if "help" in options:
        return fastcgi_help()

    try:
        import flup
    except ImportError, e:
        print >> sys.stderr, "ERROR: %s" % e
        print >> sys.stderr, "  Unable to load the flup package.  In order to run django"
        print >> sys.stderr, "  as a FastCGI application, you will need to get flup from"
        print >> sys.stderr, "  http://www.saddi.com/software/flup/   If you've already"
        print >> sys.stderr, "  installed flup, then make sure you have it in your PYTHONPATH."
        return False

    if options['method'] in ('prefork', 'fork'):
        from flup.server.fcgi_fork import WSGIServer
        wsgi_opts = {
            'maxSpare': int(options["maxspare"]),
            'minSpare': int(options["minspare"]),
            'maxChildren': int(options["maxchildren"]),
            'maxRequests': int(options["maxrequests"]),
        }
    elif options['method'] in ('thread', 'threaded'):
        from flup.server.fcgi import WSGIServer
        wsgi_opts = {
            'maxSpare': int(options["maxspare"]),
            'minSpare': int(options["minspare"]),
            'maxThreads': int(options["maxchildren"]),
        }
    else:
        return fastcgi_help("ERROR: Implementation must be one of prefork or thread.")

    # Prep up and go
    from django.core.handlers.wsgi import WSGIHandler

    if options["host"] and options["port"] and not options["socket"]:
        wsgi_opts['bindAddress'] = (options["host"], int(options["port"]))
    elif options["socket"] and not options["host"] and not options["port"]:
        wsgi_opts['bindAddress'] = options["socket"]
    elif not options["socket"] and not options["host"] and not options["port"]:
        wsgi_opts['bindAddress'] = None
    else:
        return fastcgi_help("Invalid combination of host, port, socket.")

    if options["daemonize"] is None:
        # Default to daemonizing if we're running on a socket/named pipe.
        daemonize = (wsgi_opts['bindAddress'] is not None)
    else:
        if options["daemonize"].lower() in ('true', 'yes', 't'):
            daemonize = True
        elif options["daemonize"].lower() in ('false', 'no', 'f'):
            daemonize = False
        else:
            return fastcgi_help("ERROR: Invalid option for daemonize parameter.")

    if daemonize:
        from django.utils.daemonize import become_daemon
        become_daemon(our_home_dir=options["workdir"])

    if options["pidfile"]:
        fp = open(options["pidfile"], "w")
        fp.write("%d\n" % os.getpid())
        fp.close()

    WSGIServer(WSGIHandler(), **wsgi_opts).run()

if __name__ == '__main__':
    runfastcgi(sys.argv[1:])
<|endoftext|>