<commit_before>
# generic imports
import django
import json
from lxml import etree
import math
import os
from pyspark.sql.types import StringType, StructField, StructType, BooleanType, ArrayType, IntegerType
import sys

# check for registered apps signifying readiness, if not, run django.setup() to run as standalone
if not hasattr(django, 'apps'):
	os.environ['DJANGO_SETTINGS_MODULE'] = 'combine.settings'
	sys.path.append('/opt/combine')
	django.setup()

# import django settings
from django.conf import settings
from django.db import connection

from core.models import Job, PublishedRecords

def export_records_as_xml(spark, base_path, job_dict, records_per_file):

	'''
	Function to export multiple Jobs, with folder hierarchy for each Job

	Args:
		base_path (str): base location for folder structure
		job_dict (dict): dictionary of directory name --> list of Job ids
			- e.g. single job: {'j29':[29]}
			- e.g. published records: {'foo':[2,42], 'bar':[3]}
				- in this case, a union will be performed for all Jobs within a single key
		records_per_file (int): number of XML records per file
	'''

	# clean base path
	base_path = "file:///%s" % base_path.lstrip('file://').rstrip('/')

	# loop through keys and export
	for folder_name, job_ids in job_dict.items():

		# handle single job_id
		if len(job_ids) == 1:

			# get Job records as df
			rdd_to_write = get_job_as_df(spark, job_ids[0]).select('document').rdd

		# handle multiple jobs
		else:

			rdds = [ get_job_as_df(spark, job_id).select('document').rdd for job_id in job_ids ]
			rdd_to_write = spark.sparkContext.union(rdds)

		# write rdd to disk
		rdd_to_write.repartition(math.ceil(rdd_to_write.count()/int(records_per_file)))\
		.map(lambda row: row.document.replace('<?xml version=\"1.0\" encoding=\"UTF-8\"?>',''))\
		.saveAsTextFile('%s/%s' % (base_path, folder_name))

############################################################################
# Convenience Functions
############################################################################

def get_job_as_df(spark, job_id, remove_id=False):

	'''
	Convenience method to retrieve set of records as Spark DataFrame
	'''

	pipeline = json.dumps({'$match': {'job_id': job_id}})
	mdf = spark.read.format("com.mongodb.spark.sql.DefaultSource")\
	.option("uri","mongodb://127.0.0.1")\
	.option("database","combine")\
	.option("collection","record")\
	.option("partitioner","MongoSamplePartitioner")\
	.option("spark.mongodb.input.partitionerOptions.partitionSizeMB",settings.MONGO_READ_PARTITION_SIZE_MB)\
	.option("pipeline",pipeline).load()

	# if remove ID
	if remove_id:
		mdf = mdf.select([ c for c in mdf.columns if c != '_id' ])

	return mdf

def get_job_es_as_df(spark, job_id=None, indices=None, es_query=None, field_exclude=None, id_only=False):

	'''
	Convenience method to retrieve ElasticSearch indices as DataFrame
	'''

	# handle indices
	if job_id:
		es_indexes = 'j%s' % job_id
	elif indices:
		es_indexes = ','.join(indices)

	# prep conf
	conf = {
		"es.resource":"%s/record" % es_indexes
	}

	# handle es_query
	if es_query:
		conf['es.query'] = es_query

	# handle field exclusion
	if field_exclude:
		conf['es.read.field.exclude'] = field_exclude

	# get es index as RDD
	es_rdd = spark.sparkContext.newAPIHadoopRDD(
		inputFormatClass="org.elasticsearch.hadoop.mr.EsInputFormat",
		keyClass="org.apache.hadoop.io.NullWritable",
		valueClass="org.elasticsearch.hadoop.mr.LinkedMapWritable",
		conf=conf)

	# id only
	if id_only:
		es_df = es_rdd.map(lambda row: (row[0], )).toDF()
	else:
		es_df = es_rdd.toDF()

	# return
	return es_df

def get_sql_job_as_df(spark, job_id, remove_id=False):

	sqldf = spark.read.jdbc(settings.COMBINE_DATABASE['jdbc_url'],'core_record',properties=settings.COMBINE_DATABASE)
	sqldf = sqldf.filter(sqldf['job_id'] == job_id)

	# if remove ID
	if remove_id:
		sqldf = sqldf.select([ c for c in sqldf.columns if c != 'id' ])

	return sqldf

def copy_sql_to_mongo(spark, job_id):

	# get sql job
	sdf = get_sql_job_as_df(spark, job_id, remove_id=True)

	# repartition
	sdf = sdf.rdd.repartition(200).toDF(schema=sdf.schema)

	# insert
	sdf.write.format("com.mongodb.spark.sql.DefaultSource")\
	.mode("append")\
	.option("uri","mongodb://127.0.0.1")\
	.option("database","combine")\
	.option("collection", "record").save()

def copy_sql_to_mongo_adv(spark, job_id, lowerBound, upperBound, numPartitions):

	sqldf = spark.read.jdbc(
			settings.COMBINE_DATABASE['jdbc_url'],
			'core_record',
			properties=settings.COMBINE_DATABASE,
			column='id',
			lowerBound=lowerBound,
			upperBound=upperBound,
			numPartitions=numPartitions
		)
	db_records = sqldf.filter(sqldf.job_id == int(job_id))

	db_records.write.format("com.mongodb.spark.sql.DefaultSource")\
	.mode("append")\
	.option("uri","mongodb://127.0.0.1")\
	.option("database","combine")\
	.option("collection", "record").save()

<commit_msg>console<commit_after>
# generic imports
import django
import json
from lxml import etree
import math
import os
from pyspark.sql.types import StringType, StructField, StructType, BooleanType, ArrayType, IntegerType
import sys

# check for registered apps signifying readiness, if not, run django.setup() to run as standalone
if not hasattr(django, 'apps'):
	os.environ['DJANGO_SETTINGS_MODULE'] = 'combine.settings'
	sys.path.append('/opt/combine')
	django.setup()

# import django settings
from django.conf import settings
from django.db import connection

from core.models import Job, PublishedRecords
from core.es import es_handle

def export_records_as_xml(spark, base_path, job_dict, records_per_file):

	'''
	Function to export multiple Jobs, with folder hierarchy for each Job

	Args:
		base_path (str): base location for folder structure
		job_dict (dict): dictionary of directory name --> list of Job ids
			- e.g. single job: {'j29':[29]}
			- e.g. published records: {'foo':[2,42], 'bar':[3]}
				- in this case, a union will be performed for all Jobs within a single key
		records_per_file (int): number of XML records per file
	'''

	# clean base path
	base_path = "file:///%s" % base_path.lstrip('file://').rstrip('/')

	# loop through keys and export
	for folder_name, job_ids in job_dict.items():

		# handle single job_id
		if len(job_ids) == 1:

			# get Job records as df
			rdd_to_write = get_job_as_df(spark, job_ids[0]).select('document').rdd

		# handle multiple jobs
		else:

			rdds = [ get_job_as_df(spark, job_id).select('document').rdd for job_id in job_ids ]
			rdd_to_write = spark.sparkContext.union(rdds)

		# write rdd to disk
		rdd_to_write.repartition(math.ceil(rdd_to_write.count()/int(records_per_file)))\
		.map(lambda row: row.document.replace('<?xml version=\"1.0\" encoding=\"UTF-8\"?>',''))\
		.saveAsTextFile('%s/%s' % (base_path, folder_name))

############################################################################
# Convenience Functions
############################################################################

def get_job_as_df(spark, job_id, remove_id=False):

	'''
	Convenience method to retrieve set of records as Spark DataFrame
	'''

	pipeline = json.dumps({'$match': {'job_id': job_id}})
	mdf = spark.read.format("com.mongodb.spark.sql.DefaultSource")\
	.option("uri","mongodb://127.0.0.1")\
	.option("database","combine")\
	.option("collection","record")\
	.option("partitioner","MongoSamplePartitioner")\
	.option("spark.mongodb.input.partitionerOptions.partitionSizeMB",settings.MONGO_READ_PARTITION_SIZE_MB)\
	.option("pipeline",pipeline).load()

	# if remove ID
	if remove_id:
		mdf = mdf.select([ c for c in mdf.columns if c != '_id' ])

	return mdf

def get_job_es(spark, job_id=None, indices=None, es_query=None, field_exclude=None, as_rdd=False):

	'''
	Convenience method to retrieve ElasticSearch indices as DataFrame
	'''

	# handle indices
	if job_id:
		es_indexes = 'j%s' % job_id
	elif indices:
		es_indexes = ','.join(indices)

	# prep conf
	conf = {
		"es.resource":"%s/record" % es_indexes,
		"es.output.json":"true",
		"es.input.max.docs.per.partition":"10000"
	}

	# handle es_query
	if es_query:
		conf['es.query'] = es_query

	# handle field exclusion
	if field_exclude:
		conf['es.read.field.exclude'] = field_exclude

	# get es index as RDD
	es_rdd = spark.sparkContext.newAPIHadoopRDD(
		inputFormatClass="org.elasticsearch.hadoop.mr.EsInputFormat",
		keyClass="org.apache.hadoop.io.NullWritable",
		valueClass="org.elasticsearch.hadoop.mr.LinkedMapWritable",
		conf=conf)

	# return rdd
	if as_rdd:
		return es_rdd

	# read json
	es_df = spark.read.json(es_rdd.map(lambda row: row[1]))

	# return
	return es_df

def get_sql_job_as_df(spark, job_id, remove_id=False):

	sqldf = spark.read.jdbc(settings.COMBINE_DATABASE['jdbc_url'],'core_record',properties=settings.COMBINE_DATABASE)
	sqldf = sqldf.filter(sqldf['job_id'] == job_id)

	# if remove ID
	if remove_id:
		sqldf = sqldf.select([ c for c in sqldf.columns if c != 'id' ])

	return sqldf

def copy_sql_to_mongo(spark, job_id):

	# get sql job
	sdf = get_sql_job_as_df(spark, job_id, remove_id=True)

	# repartition
	sdf = sdf.rdd.repartition(200).toDF(schema=sdf.schema)

	# insert
	sdf.write.format("com.mongodb.spark.sql.DefaultSource")\
	.mode("append")\
	.option("uri","mongodb://127.0.0.1")\
	.option("database","combine")\
	.option("collection", "record").save()

def copy_sql_to_mongo_adv(spark, job_id, lowerBound, upperBound, numPartitions):

	sqldf = spark.read.jdbc(
			settings.COMBINE_DATABASE['jdbc_url'],
			'core_record',
			properties=settings.COMBINE_DATABASE,
			column='id',
			lowerBound=lowerBound,
			upperBound=upperBound,
			numPartitions=numPartitions
		)
	db_records = sqldf.filter(sqldf.job_id == int(job_id))

	db_records.write.format("com.mongodb.spark.sql.DefaultSource")\
	.mode("append")\
	.option("uri","mongodb://127.0.0.1")\
	.option("database","combine")\
	.option("collection", "record").save()

<|endoftext|>


<commit_before>#!/usr/bin/env python
#import pandas
import sys
import psycopg2
#from sqlalchemy import create_engine
from geo import states
from geo.geolocation import GeoLocation

class SearchLocation:
  'Provides search functions for database queries'
  def __init__(self, Connection):
    self.__Connection = Connection
    self.start_year = Connection.start_year
    self.end_year = Connection.end_year
    self.__1998_linkage_query = "SELECT cmte_id FROM committee_master_%s WHERE cand_id='%s'"
    self.__oth_linkage_query = "SELECT cmte_id FROM candidate_linkage_%s WHERE cand_id='%s'"
    self.__zipcode_query = "SELECT state, latitude, longitude FROM zipcodes WHERE zip LIKE'%s%%';"
    self.__city_state_query = "SELECT DISTINCT state FROM zipcodes WHERE LOWER(primary_city) LIKE LOWER('%%%s%%');"
    self.__zipcodes_query = "SELECT zip FROM zipcodes WHERE latitude BETWEEN '%s' AND '%s'" + \
                            "AND longitude BETWEEN '%s' AND '%s' and state='%s';"
    self.__cand_zipcodes_query = "SELECT DISTINCT cand_name, cand_id, cand_pty_affiliation," + \
                                 "cand_city, cand_st FROM candidate_master_{0} WHERE cand_zip IN %s " + \
				 "OR cand_id LIKE '__{1}%%' ORDER BY cand_name;"
    self.__state_title_query = "SELECT cand_name, cand_id, cand_pty_affiliation, cand_city," + \
                               "cand_st FROM candidate_master_{0} WHERE %s LIKE UPPER('%%%s%%')" + \
			       "and %s LIKE UPPER('%%%s%%') OR cand_id LIKE '__%s%%';"
    self.__city_state_abbr_query = "SELECT DISTINCT cand_name, cand_id, cand_pty_affiliation, cand_city," +\
                              "cand_st FROM candidate_master_{0} WHERE %s LIKE UPPER('%%%s%%') OR cand_id LIKE '__%s%%';"
    self.__first_last_name_query = "SELECT cand_name, cand_id, cand_pty_affiliation, cand_city," + \
                                   "cand_st FROM candidate_master_{0} WHERE cand_name LIKE UPPER('%%%s%%')" + \
				   "AND cand_name LIKE UPPER('%%%s%%');"
    self.__cand_wo_state_query =  "SELECT cand_name, cand_id, cand_pty_affiliation, cand_city, " + \
                                  "cand_st FROM candidate_master_{0} WHERE %s LIKE UPPER('%%%s%%') "
    self.__cand_wo_state_query_suffix = "OR cand_id LIKE '__%s%%';"
    self.__name_query = "SELECT cand_name, cand_id, cand_pty_affiliation, cand_city, cand_st " + \
                        "FROM candidate_master_%s WHERE cand_name LIKE UPPER('%%%s%%');"
  def __remove_duplicates__(self, seq):
    seen = set()
    seen_add = seen.add
    return [ x for x in seq if not (x in seen or seen_add(x))]

  def __get_candidate_committees__(self, cands):
    cand_comms = {}
    for candidate in cands:
      for year in range(self.start_year, self.end_year, 2):
        if year <= 1998:
	  __linkage_query = self.__1998_linkage_query % (year, candidate[1])
        else:
	  __linkage_query =  self.__oth_linkage_query % (year, candidate[1])
        cand_comms[candidate[0]] = {"cand_id": candidate[1], "comm_ids":[]}
        self.__Connection.cur.execute(__linkage_query)
        committee_ids = self.__Connection.cur.fetchall()
        for committee_id in committee_ids:
          cand_comms[candidate[0]]["comm_ids"].append(committee_id[0])
    cand_comms = self.__remove_duplicates__(cand_comms)
    return cand_comms

  def search_names_by_zip(self, parameters):
    """Search by zipcode
    parameters = {'zipcode':zip, 'distance':distance, 'unit':measure_unit}
    'unit' must be miles or kilometers
    'distance' is any arbitrary integer value
    'zipcode' is integer or string
    """
    try:
      zipcode = parameters['zipcode']
      distance = parameters['distance']
      unit = parameters['unit']
    except KeyError:
      raise KeyError("Please define zipcode, distance, and unit")

    if unit == "miles":
      distance = distance/0.62137

    zipcode_stmt = self.__zipcode_query % zipcode
    self.__Connection.cur.execute(zipcode_stmt)
    results = self.__Connection.cur.fetchone()
    if not results:
      return False, False
    state, lat, lon = results
    loc = GeoLocation.from_degrees(lat, lon)
    SW_loc, NE_loc = loc.bounding_locations(distance)
    __zipcodes_stmt = self.__zipcodes_query % (SW_loc.deg_lat, NE_loc.deg_lat, SW_loc.deg_lon, NE_loc.deg_lon, state)
    self.__Connection.cur.execute(__zipcodes_stmt)
    zipcodes = self.__Connection.cur.fetchall()
    __zipcodes = []
    candidates = []
    for __zipcode in zipcodes:
      __zipcodes.append(__zipcode[0].split(".")[0])
    for year in range(self.start_year, self.end_year, 2):
      print state
      __candidates_query = self.__cand_zipcodes_query.format(year, state)
      print __candidates_query
      __candidates_query = self.__Connection.cur.mogrify(__candidates_query, (tuple(__zipcodes),))
      print __candidates_query
      self.__Connection.cur.execute(__candidates_query)
      candidates += self.__Connection.cur.fetchall()
    candidates = self.__remove_duplicates__(candidates)
    candidates_committees = self.__get_candidate_committees__(candidates)

    return candidates, candidates_committees

  def search_by_city_state(self, parameters):
    """Search by city, state, or city and state
    parameters = {'cand_st':state, 'cand_city': city}
    """
    st_key = 'cand_st'
    city_key = 'cand_city'

    try:
      st_query = parameters[st_key].upper()
      city_query = parameters[city_key].upper()
    except KeyError:
      raise KeyError("Please define search parameter")
    if st_query and city_query:
      print "State and city search"
      print st_query, city_query
      for state in states.states_titles:
	if state['name'].lower() == st_query.lower():
	  st_query = state['abbreviation'].upper()
      __state_query_stmt = self.__state_title_query % (city_key, city_query, st_key, st_query, st_query)

    elif st_query and not city_query:
      print "State NOT city search"
      print st_query, city_query
      for state in states.states_titles:
	if state['name'].lower() == st_query.lower():
	  st_query = state['abbreviation'].upper()
      __state_query_stmt = self.__city_state_abbr_query % (st_key, st_query, st_query)

    elif not st_query and city_query:
      print "NOT State city search"
      print st_query, city_query
      __city_state_query = self.__city_state_query % (city_query)
      self.__Connection.cur.execute(__city_state_query)
      _states = self.__Connection.cur.fetchall()
      if not states:
	for _state in states.states_titles:
	  if _state['name'].lower() == city_query.lower():
	    st_query = _state['abbreviation'].upper()
        __state_query_stmt = self.__city_state_abbr_query % (st_key, st_query, st_query)
      else:
	__state_query_stmt =  self.__cand_wo_state_query % (city_key, city_query)
	for _state in _states:
	  __state_query_stmt += self.__cand_wo_state_query_suffix % _state

    elif not st_query and not city_query:
      print "Not sure what's up here"
      print st_query, city_query
      return False, False
    candidates = []
    for year in range(self.start_year, self.end_year, 2):
      print __state_query_stmt
      self.__Connection.cur.execute(__state_query_stmt.format(str(year)))
      candidates += self.__Connection.cur.fetchall()

    candidates = self.__remove_duplicates__(candidates)
    candidates_committees= self.__get_candidate_committees__(candidates)
      # return ([(name, cand_id, cand_pty_affiliation, cand_city, cand_st), ...], {cand_name : {cand_id: 'cand_id', comm_ids: [cmte_id]}}
    return candidates, candidates_committees

  def search_by_name(self, parameters):
    """Search by name
    parameters = {'name':name}
    """
    try:
      name = parameters['name'].strip()
    except KeyError:
      raise KeyError("Please define name")
    candidates = []
    for year in range(self.start_year, self.end_year, 2):
      if len(name.split(" ")) > 1 or "," in name:
        __temp__ = name.split(" ")
        if len(__temp__) > 0:
          query_by_name = self.__first_last_name_query % (year, __temp__[0], __temp__[1])
          self.__Connection.cur.execute(__query_by_name)
          candidates = self.__Connection.cur.fetchall()
          if len(candidates) < 1:
	    query_by_name = self.__name_query % (year, __temp__[1].strip(','))
	    self.__Connection.cur.execute(__query_by_name)
	    candidates = self.__Connection.cur.fetchall()
      else:
        query_by_name = self.__name_query % (year, name)
        self.__Connection.cur.execute(query_by_name)
        candidates = self.__Connection.cur.fetchall()

    candidates = self.__remove_duplicates__(candidates)
    candidates_committees = self.__get_candidate_committees__(candidates)
    return candidates, candidates_committees
  <commit_msg>Fixed search box form name for searchByZip<commit_after>#!/usr/bin/env python
#import pandas
import sys
import psycopg2
#from sqlalchemy import create_engine
from geo import states
from geo.geolocation import GeoLocation

class SearchLocation:
  'Provides search functions for database queries'
  def __init__(self, Connection):
    self.__Connection = Connection
    self.start_year = Connection.start_year
    self.end_year = Connection.end_year
    self.__1998_linkage_query = "SELECT cmte_id FROM committee_master_%s WHERE cand_id='%s'"
    self.__oth_linkage_query = "SELECT cmte_id FROM candidate_linkage_%s WHERE cand_id='%s'"
    self.__zipcode_query = "SELECT state, latitude, longitude FROM zipcodes WHERE zip LIKE'%s%%';"
    self.__city_state_query = "SELECT DISTINCT state FROM zipcodes WHERE LOWER(primary_city) LIKE LOWER('%%%s%%');"
    self.__zipcodes_query = "SELECT zip FROM zipcodes WHERE latitude BETWEEN '%s' AND '%s'" + \
                            "AND longitude BETWEEN '%s' AND '%s' and state='%s';"
    self.__cand_zipcodes_query = "SELECT DISTINCT cand_name, cand_id, cand_pty_affiliation," + \
                                 "cand_city, cand_st FROM candidate_master_{0} WHERE cand_zip IN %s " + \
				 "OR cand_id LIKE '__{1}%%' ORDER BY cand_name;"
    self.__state_title_query = "SELECT cand_name, cand_id, cand_pty_affiliation, cand_city," + \
                               "cand_st FROM candidate_master_{0} WHERE %s LIKE UPPER('%%%s%%')" + \
			       "and %s LIKE UPPER('%%%s%%') OR cand_id LIKE '__%s%%';"
    self.__city_state_abbr_query = "SELECT DISTINCT cand_name, cand_id, cand_pty_affiliation, cand_city," +\
                              "cand_st FROM candidate_master_{0} WHERE %s LIKE UPPER('%%%s%%') OR cand_id LIKE '__%s%%';"
    self.__first_last_name_query = "SELECT cand_name, cand_id, cand_pty_affiliation, cand_city," + \
                                   "cand_st FROM candidate_master_{0} WHERE cand_name LIKE UPPER('%%%s%%')" + \
				   "AND cand_name LIKE UPPER('%%%s%%');"
    self.__cand_wo_state_query =  "SELECT cand_name, cand_id, cand_pty_affiliation, cand_city, " + \
                                  "cand_st FROM candidate_master_{0} WHERE %s LIKE UPPER('%%%s%%') "
    self.__cand_wo_state_query_suffix = "OR cand_id LIKE '__%s%%'"
    self.__name_query = "SELECT cand_name, cand_id, cand_pty_affiliation, cand_city, cand_st " + \
                        "FROM candidate_master_%s WHERE cand_name LIKE UPPER('%%%s%%');"
  def __remove_duplicates__(self, seq):
    seen = set()
    seen_add = seen.add
    return [ x for x in seq if not (x in seen or seen_add(x))]

  def __get_candidate_committees__(self, cands):
    cand_comms = {}
    for candidate in cands:
      for year in range(self.start_year, self.end_year, 2):
        if year <= 1998:
	  __linkage_query = self.__1998_linkage_query % (year, candidate[1])
        else:
	  __linkage_query =  self.__oth_linkage_query % (year, candidate[1])
        cand_comms[candidate[0]] = {"cand_id": candidate[1], "comm_ids":[]}
        self.__Connection.cur.execute(__linkage_query)
        committee_ids = self.__Connection.cur.fetchall()
        for committee_id in committee_ids:
          cand_comms[candidate[0]]["comm_ids"].append(committee_id[0])
    cand_comms = self.__remove_duplicates__(cand_comms)
    return cand_comms

  def search_names_by_zip(self, parameters):
    """Search by zipcode
    parameters = {'zipcode':zip, 'distance':distance, 'unit':measure_unit}
    'unit' must be miles or kilometers
    'distance' is any arbitrary integer value
    'zipcode' is integer or string
    """
    try:
      zipcode = parameters['zipcode']
      distance = parameters['distance']
      unit = parameters['unit']
    except KeyError:
      raise KeyError("Please define zipcode, distance, and unit")

    if unit == "miles":
      distance = distance/0.62137

    zipcode_stmt = self.__zipcode_query % zipcode
    self.__Connection.cur.execute(zipcode_stmt)
    results = self.__Connection.cur.fetchone()
    if not results:
      return False, False
    state, lat, lon = results
    loc = GeoLocation.from_degrees(lat, lon)
    SW_loc, NE_loc = loc.bounding_locations(distance)
    __zipcodes_stmt = self.__zipcodes_query % (SW_loc.deg_lat, NE_loc.deg_lat, SW_loc.deg_lon, NE_loc.deg_lon, state)
    self.__Connection.cur.execute(__zipcodes_stmt)
    zipcodes = self.__Connection.cur.fetchall()
    __zipcodes = []
    candidates = []
    for __zipcode in zipcodes:
      __zipcodes.append(__zipcode[0].split(".")[0])
    for year in range(self.start_year, self.end_year, 2):
      print state
      __candidates_query = self.__cand_zipcodes_query.format(year, state)
      print __candidates_query
      __candidates_query = self.__Connection.cur.mogrify(__candidates_query, (tuple(__zipcodes),))
      print __candidates_query
      self.__Connection.cur.execute(__candidates_query)
      candidates += self.__Connection.cur.fetchall()
    candidates = self.__remove_duplicates__(candidates)
    candidates_committees = self.__get_candidate_committees__(candidates)

    return candidates, candidates_committees

  def search_by_city_state(self, parameters):
    """Search by city, state, or city and state
    parameters = {'cand_st':state, 'cand_city': city}
    """
    st_key = 'cand_st'
    city_key = 'cand_city'

    try:
      st_query = parameters[st_key].upper()
      city_query = parameters[city_key].upper()
    except KeyError:
      raise KeyError("Please define search parameter")
    if st_query and city_query:
      print "State and city search"
      print st_query, city_query
      for state in states.states_titles:
	if state['name'].lower() == st_query.lower():
	  st_query = state['abbreviation'].upper()
      __state_query_stmt = self.__state_title_query % (city_key, city_query, st_key, st_query, st_query)

    elif st_query and not city_query:
      print "State NOT city search"
      print st_query, city_query
      for state in states.states_titles:
	if state['name'].lower() == st_query.lower():
	  st_query = state['abbreviation'].upper()
      __state_query_stmt = self.__city_state_abbr_query % (st_key, st_query, st_query)

    elif not st_query and city_query:
      print "NOT State city search"
      print st_query, city_query
      __city_state_query = self.__city_state_query % (city_query)
      self.__Connection.cur.execute(__city_state_query)
      _states = self.__Connection.cur.fetchall()
      if not states:
	for _state in states.states_titles:
	  if _state['name'].lower() == city_query.lower():
	    st_query = _state['abbreviation'].upper()
        __state_query_stmt = self.__city_state_abbr_query % (st_key, st_query, st_query)
      else:
	__state_query_stmt =  self.__cand_wo_state_query % (city_key, city_query)
	for _state in _states:
	  __state_query_stmt += self.__cand_wo_state_query_suffix % _state
        __state_query_stmt += "; ORDER BY CITY;"
    elif not st_query and not city_query:
      print "Not sure what's up here"
      print st_query, city_query
      return False, False
    candidates = []
    for year in range(self.start_year, self.end_year, 2):
      print __state_query_stmt
      self.__Connection.cur.execute(__state_query_stmt.format(str(year)))
      candidates += self.__Connection.cur.fetchall()

    candidates = self.__remove_duplicates__(candidates)
    candidates_committees= self.__get_candidate_committees__(candidates)
      # return ([(name, cand_id, cand_pty_affiliation, cand_city, cand_st), ...], {cand_name : {cand_id: 'cand_id', comm_ids: [cmte_id]}}
    return candidates, candidates_committees

  def search_by_name(self, parameters):
    """Search by name
    parameters = {'name':name}
    """
    try:
      name = parameters['name'].strip()
    except KeyError:
      raise KeyError("Please define name")
    candidates = []
    for year in range(self.start_year, self.end_year, 2):
      if len(name.split(" ")) > 1 or "," in name:
        __temp__ = name.split(" ")
        if len(__temp__) > 0:
          query_by_name = self.__first_last_name_query % (year, __temp__[0], __temp__[1])
          self.__Connection.cur.execute(__query_by_name)
          candidates = self.__Connection.cur.fetchall()
          if len(candidates) < 1:
	    query_by_name = self.__name_query % (year, __temp__[1].strip(','))
	    self.__Connection.cur.execute(__query_by_name)
	    candidates = self.__Connection.cur.fetchall()
      else:
        query_by_name = self.__name_query % (year, name)
        self.__Connection.cur.execute(query_by_name)
        candidates = self.__Connection.cur.fetchall()

    candidates = self.__remove_duplicates__(candidates)
    candidates_committees = self.__get_candidate_committees__(candidates)
    return candidates, candidates_committees
  <|endoftext|>


<commit_before>"""Cosmic Ray version info."""

__version_info__ = (3, 0, 6)
__version__ = '.'.join(map(str, __version_info__))
<commit_msg>Bumped version to 3.1.0.<commit_after>"""Cosmic Ray version info."""

__version_info__ = (3, 1, 0)
__version__ = '.'.join(map(str, __version_info__))
<|endoftext|>


<commit_before>from couchexport.schema import get_docs, get_schema, get_schema_new
import csv
import json
import zipfile
from StringIO import StringIO
from django.conf import settings
from couchexport.models import ExportSchema, Format
from couchdbkit.client import Database
import re
from dimagi.utils.mixins import UnicodeMixIn
from django.template.loader import render_to_string
from couchdbkit.consumer import Consumer
from dimagi.utils.couch.database import get_db
from couchexport.writers import Excel2007ExportWriter, CsvExportWriter,\
    Excel2003ExportWriter, JsonExportWriter, HtmlExportWriter

class ExportConfiguration(object):
    """
    A representation of the configuration parameters for an export and
    some functions to actually facilitate the export from this config.
    """

    def __init__(self, database, schema_index, previous_export=None, filter=None):
        self.database = database
        self.schema_index = schema_index
        self.previous_export = previous_export
        self.filter = filter
        self.current_seq = self.database.info()["update_seq"]
        self.potentially_relevant_ids = self._potentially_relevant_ids()

    def include(self, document):
        """
        Returns True if the document should be included in the results,
        otherwise false
        """
        return self.filter(document) if self.filter else True

    def _all_ids(self):
        """
        Gets view results for all documents matching this schema
        """
        return [result['id'] for result in \
                self.database.view("couchexport/schema_index",
                                   key=self.schema_index).all()]

    def _potentially_relevant_ids(self):
        if self.previous_export is not None:
            consumer = Consumer(self.database)
            view_results = consumer.fetch(since=self.previous_export.seq)
            if view_results:
                try:
                    include_ids = set([res["id"] for res in view_results["results"]])
                    possible_ids = set(self._all_ids())
                    return list(include_ids.intersection(possible_ids))
                except Exception:
                    import logging
                    logging.exception("export failed! results: %s" % view_results)
                    raise
            else:
                # sometimes this comes back empty. I think it might be a bug
                # in couchdbkit, but it's impossible to consistently reproduce.
                # For now, just assume this is fine.
                return []
        else:
            return self._all_ids()

    def get_docs(self):
        for doc_id in self.potentially_relevant_ids:
            doc = self.database.get(doc_id)
            if self.include(doc):
                yield doc

    def last_checkpoint(self):
        return self.previous_export or ExportSchema.last(self.schema_index)

def get_writer(format):
    if format == Format.CSV:
        return CsvExportWriter()
    elif format == Format.HTML:
        return HtmlExportWriter()
    elif format == Format.JSON:
        return JsonExportWriter()
    elif format == Format.XLS:
        return Excel2003ExportWriter()
    elif format == Format.XLS_2007:
        return Excel2007ExportWriter()
    else:
        raise Exception("Unsupported export format: %s!" % format)

def export_from_tables(tables, file, format, max_column_size=2000):
    if format == Format.CSV:
        _export_csv(tables, file, max_column_size)
    elif format == Format.HTML:
        _export_html(tables, file, max_column_size)
    elif format == Format.JSON:
        _export_json(tables, file)
    elif format == Format.XLS:
        _export_excel(tables, max_column_size).save(file)
    elif format == Format.XLS_2007:
        _export_excel_2007(tables, max_column_size).save(file)
    else:
        raise Exception("Unsupported export format: %s!" % format)

def export(schema_index, file, format=Format.XLS_2007,
           previous_export_id=None, filter=None,
           max_column_size=2000, separator='|'):
    """
    Exports data from couch documents matching a given tag to a file.
    Returns true if it finds data, otherwise nothing
    """

    previous_export = ExportSchema.get(previous_export_id) \
                      if previous_export_id else None
    database = get_db()
    config = ExportConfiguration(database, schema_index,
                                 previous_export, filter)

    # get and checkpoint the latest schema
    updated_schema = get_schema_new(config)
    export_schema_checkpoint = ExportSchema(seq=config.current_seq,
                                            schema=updated_schema,
                                            index=config.schema_index)
    export_schema_checkpoint.save()

    # transform docs onto output and save
    writer = get_writer(format)

    # open the doc and the headers
    formatted_headers = get_headers(updated_schema, separator=separator)
    writer.open(formatted_headers, file)

    for doc in config.get_docs():
        writer.write(format_tables(create_intermediate_tables(doc, updated_schema),
                                   include_headers=False, separator=separator))
    writer.close()
    return export_schema_checkpoint

class Constant(UnicodeMixIn):
    def __init__(self, message):
        self.message = message
    def __unicode__(self):
        return self.message

SCALAR_NEVER_WAS = settings.COUCHEXPORT_SCALAR_NEVER_WAS \
                    if hasattr(settings, "COUCHEXPORT_SCALAR_NEVER_WAS") \
                    else "---"

LIST_NEVER_WAS = settings.COUCHEXPORT_LIST_NEVER_WAS \
                    if hasattr(settings, "COUCHEXPORT_LIST_NEVER_WAS") \
                    else "this list never existed"

scalar_never_was = Constant(SCALAR_NEVER_WAS)
list_never_was = Constant(LIST_NEVER_WAS)

def render_never_was(schema):
    if isinstance(schema, dict):
        answ = {}
        for key in schema:
            answ[key] = render_never_was(schema[key])
        return answ

    elif isinstance(schema, list):
        return list_never_was
    else:
        return scalar_never_was

unknown_type = None
def fit_to_schema(doc, schema):

    def log(msg):
        raise Exception("doc-schema mismatch: %s" % msg)

    if schema is None:
        if doc:
            log("%s is not null" % doc)
        return None
    if isinstance(schema, list):
        if not doc:
            doc = []
        if not isinstance(doc, list):
            return fit_to_schema([doc], schema)
        answ = []
        schema_, = schema
        for doc_ in doc:
            answ.append(fit_to_schema(doc_, schema_))
        return answ
    if isinstance(schema, dict):
        if not doc:
            doc = {}
        doc_keys = set(doc.keys())
        schema_keys = set(schema.keys())
        if doc_keys - schema_keys:
            log("doc has keys not in schema: %s" % (', '.join(doc_keys - schema_keys)))
        answ = {}
        for key in schema:
            #if schema[key] == unknown_type: continue
            if doc.has_key(key):
                answ[key] = fit_to_schema(doc.get(key), schema[key])
            else:
                answ[key] = render_never_was(schema[key])
        return answ
    if schema == "string":
        if not doc:
            doc = ""
        if not isinstance(doc, basestring):
        #log("%s is not a string" % doc)
            doc = unicode(doc)
        return doc

def get_headers(schema, separator="|"):
    return format_tables(create_intermediate_tables(schema, schema),
                             include_data=False, separator=separator)

def create_intermediate_tables(docs, schema, integer='#'):
    def lookup(doc, keys):
        for key in keys:
            doc = doc[key]
        return doc
    def split_path(path):
        table = []
        column = []
        id = []
        for k in path:
            if isinstance(k, basestring):
                column.append(k)
            else:
                table.extend(column)
                table.append(integer)
                column = []
                id.append(k)
        return (tuple(table), tuple(column), tuple(id))
    schema = [schema]
    docs = fit_to_schema(docs, schema)
    # first, flatten documents
    queue = [()]
    leaves = []
    while queue:
        path = queue.pop()
        d = lookup(docs, path)
        if isinstance(d, dict):
            for key in d:
                queue.append(path + (key,))
        elif isinstance(d, list):
            for i,_ in enumerate(d):
                queue.append(path + (i,))
        elif d != list_never_was:
            leaves.append((split_path(path), d))
    leaves.sort()
    tables = {}
    for (table_name, column_name, id), val in leaves:
        table = tables.setdefault(table_name, {})
        row = table.setdefault(id, {})
        row[column_name] = val

    return tables

def nice(column_name):
    return "/".join(column_name)

class FormattedRow(object):
    """
    Simple data structure to represent a row of an export. Just
    a pairing of an id and the data.

    The id should be an iterable (compound ids are supported).
    """
    def __init__(self, data, id=None, separator="."):
        self.data = data
        self.id = id
        self.separator = separator

    def has_id(self):
        return self.id is not None

    @property
    def formatted_id(self):
        if isinstance(self.id, basestring):
            return self.id
        return self.separator.join(map(unicode, self.id))

    def get_data(self):
        if self.has_id():
            yield self.formatted_id
        for val in self.data:
            yield val

def format_tables(tables, id_label='id', separator='.', include_headers=True,
                      include_data=True):
    answ = []
    assert include_data or include_headers, "This method is pretty useless if you don't include anything!"

    for table_name, table in sorted(tables.items()):
        new_table = []
        keys = sorted(table.items()[0][1].keys()) # the keys for every row are the same

        if include_headers:
            header_vals = [separator.join(key) for key in keys]
            new_table.append(FormattedRow(header_vals, [id_label], separator))

        if include_data:
            for id, row in sorted(table.items()):
                values = [row[key] for key in keys]
                new_table.append(FormattedRow(values, id, separator))

        answ.append((separator.join(table_name), new_table))
    return answ

def _export_csv(tables, file, max_column_size):
    #temp = tempfile.TemporaryFile()
    temp = file
    archive = zipfile.ZipFile(temp, 'w', zipfile.ZIP_DEFLATED)

    # write forms
    used_names = []
    for table_name, table in tables:
        used_headers = []
        table_name_truncated = _clean_name(_next_unique(table_name, used_names))
        used_names.append(table_name_truncated)
        table_file = StringIO()
        writer = csv.writer(table_file, dialect=csv.excel)
        def _truncate(val):
            ret = _next_unique(val, used_headers, max_column_size)
            used_headers.append(ret)
            return ret

        for rowcount, row in enumerate(table):
            if rowcount == 0:
                # make sure we trim the headers
                row = map(_truncate, row)
            for i, val in enumerate(row):
                if isinstance(val, unicode):
                    row[i] = val.encode("utf8")
            writer.writerow(row)
        archive.writestr("%s.csv" % table_name_truncated, table_file.getvalue())

    archive.close()
    temp.seek(0)
    return temp

def _export_html(tables, file, max_column_size):
    file.write(render_to_string("couchexport/html_export.html", {'tables': tables}))

class ConstantEncoder(json.JSONEncoder):
    def default(self, obj):
        if isinstance(obj, Constant):
            return obj.message
        return json.JSONEncoder.default(self, obj)

def _export_json(tables, file):
    new_tables = dict()
    for table in tables:
        new_tables[table[0]] = {"headers":table[1][0], "rows": table[1][1:]}

    file.write(json.dumps(new_tables, cls=ConstantEncoder))

def _export_excel(tables, max_column_size):
    try:
        import xlwt
    except ImportError:
        raise Exception("It doesn't look like this machine is configured for "
                        "excel export. To export to excel you have to run the "
                        "command:  easy_install xlutils")
    book = xlwt.Workbook()
    used_names = []
    for table_name, table in tables:
        used_headers = []
        def _truncate(val):
            ret = _next_unique(val, used_headers, max_column_size)
            used_headers.append(ret)
            return ret

        # this is in case the first 20 characters are the same, but we
        # should do something smarter.
        table_name_truncated = _next_unique(table_name, used_names, 20)
        used_names.append(table_name_truncated)
        sheet = book.add_sheet(_clean_name(table_name_truncated))

        for i,row in enumerate(table):
            if i == 0:
                # make sure we trim the headers
                row = map(_truncate, row)
            for j,val in enumerate(row):
                sheet.write(i,j,unicode(val))
    return book

def _export_excel_2007(tables, max_column_size):
    try:
        import openpyxl
    except ImportError:
        raise Exception("It doesn't look like this machine is configured for "
                        "excel export. To export to excel you have to run the "
                        "command:  easy_install openpyxl")
    book = openpyxl.workbook.Workbook()
    book.remove_sheet(book.worksheets[0])
    used_names = []
    for table_name, table in tables:
        used_headers = []
        def _truncate(val):
            ret = _next_unique(val, used_headers, max_column_size)
            used_headers.append(ret)
            return ret

        # this is in case the first 20 characters are the same, but we
        # should do something smarter.
        table_name_truncated = _next_unique(table_name, used_names, 31)
        used_names.append(table_name_truncated)
        sheet = book.create_sheet()
        sheet.title = _clean_name(table_name_truncated)
        for i,row in enumerate(table):
            if i == 0:
                # make sure we trim the headers
                row = map(_truncate, row)
            # the docs claim this should work but the source claims it doesn't
            #sheet.append(row)
            for j,val in enumerate(row):
                sheet.cell(row=i,column=j).value = unicode(val)
    return book

def _clean_name(name):
    return re.sub(r"[[\\?*/:\]]", "-", name)

def _next_unique(string, reserved_strings, max_len=500):
    counter = 1
    if len(string) > max_len:
        # truncate from the beginning since the end has more specific information
        string = string[-max_len:]
    orig_string = string
    while string in reserved_strings:
        string = "%s%s" % (orig_string, counter)
        if len(string) > max_len:
            counterlen = len(str(counter))
            string = "%s%s" % (orig_string[-(max_len - counterlen):], counter)
        counter = counter + 1
    return string

<commit_msg>better handling of empty exports<commit_after>from couchexport.schema import get_docs, get_schema, get_schema_new
import csv
import json
import zipfile
from StringIO import StringIO
from django.conf import settings
from couchexport.models import ExportSchema, Format
from couchdbkit.client import Database
import re
from dimagi.utils.mixins import UnicodeMixIn
from django.template.loader import render_to_string
from couchdbkit.consumer import Consumer
from dimagi.utils.couch.database import get_db
from couchexport.writers import Excel2007ExportWriter, CsvExportWriter,\
    Excel2003ExportWriter, JsonExportWriter, HtmlExportWriter

class ExportConfiguration(object):
    """
    A representation of the configuration parameters for an export and
    some functions to actually facilitate the export from this config.
    """

    def __init__(self, database, schema_index, previous_export=None, filter=None):
        self.database = database
        self.schema_index = schema_index
        self.previous_export = previous_export
        self.filter = filter
        self.current_seq = self.database.info()["update_seq"]
        self.potentially_relevant_ids = self._potentially_relevant_ids()

    def include(self, document):
        """
        Returns True if the document should be included in the results,
        otherwise false
        """
        return self.filter(document) if self.filter else True

    def _all_ids(self):
        """
        Gets view results for all documents matching this schema
        """
        return [result['id'] for result in \
                self.database.view("couchexport/schema_index",
                                   key=self.schema_index).all()]

    def _potentially_relevant_ids(self):
        if self.previous_export is not None:
            consumer = Consumer(self.database)
            view_results = consumer.fetch(since=self.previous_export.seq)
            if view_results:
                try:
                    include_ids = set([res["id"] for res in view_results["results"]])
                    possible_ids = set(self._all_ids())
                    return list(include_ids.intersection(possible_ids))
                except Exception:
                    import logging
                    logging.exception("export failed! results: %s" % view_results)
                    raise
            else:
                # sometimes this comes back empty. I think it might be a bug
                # in couchdbkit, but it's impossible to consistently reproduce.
                # For now, just assume this is fine.
                return []
        else:
            return self._all_ids()

    def get_docs(self):
        for doc_id in self.potentially_relevant_ids:
            doc = self.database.get(doc_id)
            if self.include(doc):
                yield doc

    def last_checkpoint(self):
        return self.previous_export or ExportSchema.last(self.schema_index)

def get_writer(format):
    if format == Format.CSV:
        return CsvExportWriter()
    elif format == Format.HTML:
        return HtmlExportWriter()
    elif format == Format.JSON:
        return JsonExportWriter()
    elif format == Format.XLS:
        return Excel2003ExportWriter()
    elif format == Format.XLS_2007:
        return Excel2007ExportWriter()
    else:
        raise Exception("Unsupported export format: %s!" % format)

def export_from_tables(tables, file, format, max_column_size=2000):
    if format == Format.CSV:
        _export_csv(tables, file, max_column_size)
    elif format == Format.HTML:
        _export_html(tables, file, max_column_size)
    elif format == Format.JSON:
        _export_json(tables, file)
    elif format == Format.XLS:
        _export_excel(tables, max_column_size).save(file)
    elif format == Format.XLS_2007:
        _export_excel_2007(tables, max_column_size).save(file)
    else:
        raise Exception("Unsupported export format: %s!" % format)

def export(schema_index, file, format=Format.XLS_2007,
           previous_export_id=None, filter=None,
           max_column_size=2000, separator='|'):
    """
    Exports data from couch documents matching a given tag to a file.
    Returns true if it finds data, otherwise nothing
    """

    previous_export = ExportSchema.get(previous_export_id) \
                      if previous_export_id else None
    database = get_db()
    config = ExportConfiguration(database, schema_index,
                                 previous_export, filter)

    # handle empty case
    if not config.potentially_relevant_ids:
        return None

    # get and checkpoint the latest schema
    updated_schema = get_schema_new(config)
    export_schema_checkpoint = ExportSchema(seq=config.current_seq,
                                            schema=updated_schema,
                                            index=config.schema_index)
    export_schema_checkpoint.save()

    # transform docs onto output and save
    writer = get_writer(format)

    # open the doc and the headers
    formatted_headers = get_headers(updated_schema, separator=separator)
    writer.open(formatted_headers, file)

    for doc in config.get_docs():
        writer.write(format_tables(create_intermediate_tables(doc, updated_schema),
                                   include_headers=False, separator=separator))
    writer.close()
    return export_schema_checkpoint

class Constant(UnicodeMixIn):
    def __init__(self, message):
        self.message = message
    def __unicode__(self):
        return self.message

SCALAR_NEVER_WAS = settings.COUCHEXPORT_SCALAR_NEVER_WAS \
                    if hasattr(settings, "COUCHEXPORT_SCALAR_NEVER_WAS") \
                    else "---"

LIST_NEVER_WAS = settings.COUCHEXPORT_LIST_NEVER_WAS \
                    if hasattr(settings, "COUCHEXPORT_LIST_NEVER_WAS") \
                    else "this list never existed"

scalar_never_was = Constant(SCALAR_NEVER_WAS)
list_never_was = Constant(LIST_NEVER_WAS)

def render_never_was(schema):
    if isinstance(schema, dict):
        answ = {}
        for key in schema:
            answ[key] = render_never_was(schema[key])
        return answ

    elif isinstance(schema, list):
        return list_never_was
    else:
        return scalar_never_was

unknown_type = None
def fit_to_schema(doc, schema):

    def log(msg):
        raise Exception("doc-schema mismatch: %s" % msg)

    if schema is None:
        if doc:
            log("%s is not null" % doc)
        return None
    if isinstance(schema, list):
        if not doc:
            doc = []
        if not isinstance(doc, list):
            return fit_to_schema([doc], schema)
        answ = []
        schema_, = schema
        for doc_ in doc:
            answ.append(fit_to_schema(doc_, schema_))
        return answ
    if isinstance(schema, dict):
        if not doc:
            doc = {}
        doc_keys = set(doc.keys())
        schema_keys = set(schema.keys())
        if doc_keys - schema_keys:
            log("doc has keys not in schema: %s" % (', '.join(doc_keys - schema_keys)))
        answ = {}
        for key in schema:
            #if schema[key] == unknown_type: continue
            if doc.has_key(key):
                answ[key] = fit_to_schema(doc.get(key), schema[key])
            else:
                answ[key] = render_never_was(schema[key])
        return answ
    if schema == "string":
        if not doc:
            doc = ""
        if not isinstance(doc, basestring):
        #log("%s is not a string" % doc)
            doc = unicode(doc)
        return doc

def get_headers(schema, separator="|"):
    return format_tables(create_intermediate_tables(schema, schema),
                             include_data=False, separator=separator)

def create_intermediate_tables(docs, schema, integer='#'):
    def lookup(doc, keys):
        for key in keys:
            doc = doc[key]
        return doc
    def split_path(path):
        table = []
        column = []
        id = []
        for k in path:
            if isinstance(k, basestring):
                column.append(k)
            else:
                table.extend(column)
                table.append(integer)
                column = []
                id.append(k)
        return (tuple(table), tuple(column), tuple(id))
    schema = [schema]
    docs = fit_to_schema(docs, schema)
    # first, flatten documents
    queue = [()]
    leaves = []
    while queue:
        path = queue.pop()
        d = lookup(docs, path)
        if isinstance(d, dict):
            for key in d:
                queue.append(path + (key,))
        elif isinstance(d, list):
            for i,_ in enumerate(d):
                queue.append(path + (i,))
        elif d != list_never_was:
            leaves.append((split_path(path), d))
    leaves.sort()
    tables = {}
    for (table_name, column_name, id), val in leaves:
        table = tables.setdefault(table_name, {})
        row = table.setdefault(id, {})
        row[column_name] = val

    return tables

def nice(column_name):
    return "/".join(column_name)

class FormattedRow(object):
    """
    Simple data structure to represent a row of an export. Just
    a pairing of an id and the data.

    The id should be an iterable (compound ids are supported).
    """
    def __init__(self, data, id=None, separator="."):
        self.data = data
        self.id = id
        self.separator = separator

    def has_id(self):
        return self.id is not None

    @property
    def formatted_id(self):
        if isinstance(self.id, basestring):
            return self.id
        return self.separator.join(map(unicode, self.id))

    def get_data(self):
        if self.has_id():
            yield self.formatted_id
        for val in self.data:
            yield val

def format_tables(tables, id_label='id', separator='.', include_headers=True,
                      include_data=True):
    answ = []
    assert include_data or include_headers, "This method is pretty useless if you don't include anything!"

    for table_name, table in sorted(tables.items()):
        new_table = []
        keys = sorted(table.items()[0][1].keys()) # the keys for every row are the same

        if include_headers:
            header_vals = [separator.join(key) for key in keys]
            new_table.append(FormattedRow(header_vals, [id_label], separator))

        if include_data:
            for id, row in sorted(table.items()):
                values = [row[key] for key in keys]
                new_table.append(FormattedRow(values, id, separator))

        answ.append((separator.join(table_name), new_table))
    return answ

def _export_csv(tables, file, max_column_size):
    #temp = tempfile.TemporaryFile()
    temp = file
    archive = zipfile.ZipFile(temp, 'w', zipfile.ZIP_DEFLATED)

    # write forms
    used_names = []
    for table_name, table in tables:
        used_headers = []
        table_name_truncated = _clean_name(_next_unique(table_name, used_names))
        used_names.append(table_name_truncated)
        table_file = StringIO()
        writer = csv.writer(table_file, dialect=csv.excel)
        def _truncate(val):
            ret = _next_unique(val, used_headers, max_column_size)
            used_headers.append(ret)
            return ret

        for rowcount, row in enumerate(table):
            if rowcount == 0:
                # make sure we trim the headers
                row = map(_truncate, row)
            for i, val in enumerate(row):
                if isinstance(val, unicode):
                    row[i] = val.encode("utf8")
            writer.writerow(row)
        archive.writestr("%s.csv" % table_name_truncated, table_file.getvalue())

    archive.close()
    temp.seek(0)
    return temp

def _export_html(tables, file, max_column_size):
    file.write(render_to_string("couchexport/html_export.html", {'tables': tables}))

class ConstantEncoder(json.JSONEncoder):
    def default(self, obj):
        if isinstance(obj, Constant):
            return obj.message
        return json.JSONEncoder.default(self, obj)

def _export_json(tables, file):
    new_tables = dict()
    for table in tables:
        new_tables[table[0]] = {"headers":table[1][0], "rows": table[1][1:]}

    file.write(json.dumps(new_tables, cls=ConstantEncoder))

def _export_excel(tables, max_column_size):
    try:
        import xlwt
    except ImportError:
        raise Exception("It doesn't look like this machine is configured for "
                        "excel export. To export to excel you have to run the "
                        "command:  easy_install xlutils")
    book = xlwt.Workbook()
    used_names = []
    for table_name, table in tables:
        used_headers = []
        def _truncate(val):
            ret = _next_unique(val, used_headers, max_column_size)
            used_headers.append(ret)
            return ret

        # this is in case the first 20 characters are the same, but we
        # should do something smarter.
        table_name_truncated = _next_unique(table_name, used_names, 20)
        used_names.append(table_name_truncated)
        sheet = book.add_sheet(_clean_name(table_name_truncated))

        for i,row in enumerate(table):
            if i == 0:
                # make sure we trim the headers
                row = map(_truncate, row)
            for j,val in enumerate(row):
                sheet.write(i,j,unicode(val))
    return book

def _export_excel_2007(tables, max_column_size):
    try:
        import openpyxl
    except ImportError:
        raise Exception("It doesn't look like this machine is configured for "
                        "excel export. To export to excel you have to run the "
                        "command:  easy_install openpyxl")
    book = openpyxl.workbook.Workbook()
    book.remove_sheet(book.worksheets[0])
    used_names = []
    for table_name, table in tables:
        used_headers = []
        def _truncate(val):
            ret = _next_unique(val, used_headers, max_column_size)
            used_headers.append(ret)
            return ret

        # this is in case the first 20 characters are the same, but we
        # should do something smarter.
        table_name_truncated = _next_unique(table_name, used_names, 31)
        used_names.append(table_name_truncated)
        sheet = book.create_sheet()
        sheet.title = _clean_name(table_name_truncated)
        for i,row in enumerate(table):
            if i == 0:
                # make sure we trim the headers
                row = map(_truncate, row)
            # the docs claim this should work but the source claims it doesn't
            #sheet.append(row)
            for j,val in enumerate(row):
                sheet.cell(row=i,column=j).value = unicode(val)
    return book

def _clean_name(name):
    return re.sub(r"[[\\?*/:\]]", "-", name)

def _next_unique(string, reserved_strings, max_len=500):
    counter = 1
    if len(string) > max_len:
        # truncate from the beginning since the end has more specific information
        string = string[-max_len:]
    orig_string = string
    while string in reserved_strings:
        string = "%s%s" % (orig_string, counter)
        if len(string) > max_len:
            counterlen = len(str(counter))
            string = "%s%s" % (orig_string[-(max_len - counterlen):], counter)
        counter = counter + 1
    return string

<|endoftext|>


<commit_before>version = (0, 5, 4)

__version__ = '.'.join(map(str, version))
<commit_msg>Bump to 0.5.5<commit_after>version = (0, 5, 5)

__version__ = '.'.join(map(str, version))
<|endoftext|>


<commit_before>
import re

class ElementParserSection:
    def __init__(self):
        self._p = {
                'DBCSR': dict(),
                'CP2K': dict(),
                'GLOBAL': dict(),
                'ENERGY': dict()
                }
        self._k = ''
        self._v = ''

    # returns true if parser can parse this line
    def match(self, line):
        m = re.match('^ (?P<key>[a-zA-Z0-9]+)\| (?P<value>.*)', line)
        if m:
            self._k = m.group('key')
            self._v = m.group('value')
            return True

        return False

    def parse(self, line):
        if self._k in self._p.keys():
            ms = re.match('^(?P<desc>.+?)(?:\s{2,})(?P<value>.+)', self._v)
            if ms:
                # add the found desc/parameter, but strip ':' beforehand
                self._p[self._k][ms.group('desc').rstrip(':')] = ms.group('value')
            else:
                raise InvalidValueForKey(self._k, self._v)

    # this element parser is stateless, always finish directly
    def finished(self):
        return True

    def data(self):
        return self._p

class ElementParserProgramInfo:
    def __init__(self):
        self._k = ''
        self._v = ''
        self._finished = True
        self._p = {}

    # returns true if parser can parse this line
    def match(self, line):
        # if we are here, we finished parsing before
        self._finished = True

        m = re.match('^ [\* ]{13,} PROGRAM (?P<key>(STARTED (AT|ON|BY|IN))|(RAN (ON|BY))|STOPPED IN|PROCESS ID|ENDED AT)\s+(?P<value>.*)$', line)

        if m:
            self._k = m.group('key')
            self._v = m.group('value')
            return True

        return False

    def parse(self, line):
        if self._finished:
            self._p[self._k] = self._v

            if self._k in ['STARTED IN', 'STOPPED IN']:
                # we have to check the next line first
                self._finished = False

        # possible second line of the value
        else:
            mp = re.match(' \s{42}(?P<value>.+)', line)
            if mp:
                # append the value to the existing one and return True to get another peek,
                # we might have yet another line after all!
                self._p[self._k] += mp.group('value').strip()
                return True

            # give other parsers a chance with this line, but we are done here
            self._finished = True
            return False

    def finished(self):
        return self._finished

    def data(self):
        return { 'PROGRAM': self._p }

class ElementParserError:
    # returns true if parser can parse this line
    def match(self, line):
        m = re.match('^ \*{76}$', line)

        if m:
            return True

        return False

    def parse(self, line):
        if self._k in self._p.keys():
            ms = re.match('^(?P<desc>.+?)(?:\s{2,})(?P<value>.+)', self._value)
            if ms:
                # add the found desc/parameter, but strip ':' beforehand
                self._p[self._k][ms.group('desc').rstrip(':')] = ms.group('value')
            else:
                raise InvalidValueForKey(self._k, self._value)

    # this element parser is stateless, always finish directly
    def finished(self):
        return True

    def data(self):
        return {}

class ElementParserTable:
    def __init__(self):
        self._tn = ''

    # returns true if parser can parse this line
    def match(self, line):
        m = re.match('^ \*\*\* (?P<tablename>.+?) \*\*\*$', line)

        if m:
            self._tn = m.group('tablename')
            return True

        return False

    def parse(self, line):
        pass

    # this element parser is stateless, always finish directly
    def finished(self):
        return True

    def data(self):
        return {}

class CP2KOutputParser:
    def __init__(self):
        self._parsers = [
                ElementParserSection(),
                ElementParserError(),
                ElementParserProgramInfo(),
                ]

    def parse(self, fh):
        p = None
        for line in fh:
            line = line.strip('\n')

            # if we still have an element parser set and it is not done yet, continue with that one
            if p is not None and not p.finished():
                # in case the parser needed a lookahead but was in fact done, it will return false
                if p.parse(line):
                    continue

            # otherwise restart with all possible parsers
            for p in self._parsers:
                if p.match(line):
                    p.parse(line)
                    break
#            else:
#                if len(line.strip()) > 0:
#                    print('no parser found for: "%s"' % line)

    # the query language is supposed to follow the one from the 'jq' tool,
    # but for now we support only '.' (for everything), '.foo.bar' and
    def query(self, q):
        keys = q.split('.')

        # the string should start with a '.', so the first segment is always empty
        # TODO: throw exception here
        if keys[0]:
            return {}

        d = {}

        # merge the data of all parsers first
        for p in self._parsers:
            for k, v in p.data().items():
                d[k] = v

        # if the second key is empty, '.' was passed and we return everything
        if not keys[1] and len(keys) is 2:
            return d

        # the first key is always in one of the parsers
        try:
            for k in keys[1:]:
                m = re.match(r'(?P<k>[^\[\]]*)(\[(?P<i>[\d\[\]]+)\])?$', k)
                if m.group('k') is not None:
                    d = d[m.group('k')]
                if m.group('i') is not None:
                    for i in m.group('i').split(']['):
                        # for dictionaries sort it alphabetically first (which makes it automatically a list of pairs)
                        if type(d) is dict:
                            d = sorted(d.items())
                        d = d[int(i)]

        # ignore key or type errors (when walking the parser data objects)
        except (TypeError, KeyError, IndexError):
            return {}

        return d
<commit_msg>parser: move import into class instead of global<commit_after>
class ElementParserSection:
    import re

    def __init__(self):
        self._p = {
                'DBCSR': dict(),
                'CP2K': dict(),
                'GLOBAL': dict(),
                'ENERGY': dict()
                }
        self._k = ''
        self._v = ''

    # returns true if parser can parse this line
    def match(self, line):
        m = re.match('^ (?P<key>[a-zA-Z0-9]+)\| (?P<value>.*)', line)
        if m:
            self._k = m.group('key')
            self._v = m.group('value')
            return True

        return False

    def parse(self, line):
        if self._k in self._p.keys():
            ms = re.match('^(?P<desc>.+?)(?:\s{2,})(?P<value>.+)', self._v)
            if ms:
                # add the found desc/parameter, but strip ':' beforehand
                self._p[self._k][ms.group('desc').rstrip(':')] = ms.group('value')
            else:
                raise InvalidValueForKey(self._k, self._v)

    # this element parser is stateless, always finish directly
    def finished(self):
        return True

    def data(self):
        return self._p

class ElementParserProgramInfo:
    import re

    def __init__(self):
        self._k = ''
        self._v = ''
        self._finished = True
        self._p = {}

    # returns true if parser can parse this line
    def match(self, line):
        # if we are here, we finished parsing before
        self._finished = True

        m = re.match('^ [\* ]{13,} PROGRAM (?P<key>(STARTED (AT|ON|BY|IN))|(RAN (ON|BY))|STOPPED IN|PROCESS ID|ENDED AT)\s+(?P<value>.*)$', line)

        if m:
            self._k = m.group('key')
            self._v = m.group('value')
            return True

        return False

    def parse(self, line):
        if self._finished:
            self._p[self._k] = self._v

            if self._k in ['STARTED IN', 'STOPPED IN']:
                # we have to check the next line first
                self._finished = False

        # possible second line of the value
        else:
            mp = re.match(' \s{42}(?P<value>.+)', line)
            if mp:
                # append the value to the existing one and return True to get another peek,
                # we might have yet another line after all!
                self._p[self._k] += mp.group('value').strip()
                return True

            # give other parsers a chance with this line, but we are done here
            self._finished = True
            return False

    def finished(self):
        return self._finished

    def data(self):
        return {'PROGRAM': self._p}

class ElementParserError:
    import re

    # returns true if parser can parse this line
    def match(self, line):
        m = re.match('^ \*{76}$', line)

        if m:
            return True

        return False

    def parse(self, line):
        if self._k in self._p.keys():
            ms = re.match('^(?P<desc>.+?)(?:\s{2,})(?P<value>.+)', self._value)
            if ms:
                # add the found desc/parameter, but strip ':' beforehand
                self._p[self._k][ms.group('desc').rstrip(':')] = ms.group('value')
            else:
                raise InvalidValueForKey(self._k, self._value)

    # this element parser is stateless, always finish directly
    def finished(self):
        return True

    def data(self):
        return {}

class ElementParserTable:
    import re

    def __init__(self):
        self._tn = ''

    # returns true if parser can parse this line
    def match(self, line):
        m = re.match('^ \*\*\* (?P<tablename>.+?) \*\*\*$', line)

        if m:
            self._tn = m.group('tablename')
            return True

        return False

    def parse(self, line):
        pass

    # this element parser is stateless, always finish directly
    def finished(self):
        return True

    def data(self):
        return {}

class CP2KOutputParser:
    def __init__(self):
        self._parsers = [
                ElementParserSection(),
                ElementParserError(),
                ElementParserProgramInfo(),
                ]

    def parse(self, fh):
        p = None
        for line in fh:
            line = line.strip('\n')

            # if we still have an element parser set and it is not done yet, continue with that one
            if p is not None and not p.finished():
                # in case the parser needed a lookahead but was in fact done, it will return false
                if p.parse(line):
                    continue

            # otherwise restart with all possible parsers
            for p in self._parsers:
                if p.match(line):
                    p.parse(line)
                    break
#            else:
#                if len(line.strip()) > 0:
#                    print('no parser found for: "%s"' % line)

    # the query language is supposed to follow the one from the 'jq' tool,
    # but for now we support only '.' (for everything), '.foo.bar' and
    def query(self, q):
        import re

        keys = q.split('.')

        # the string should start with a '.', so the first segment is always empty
        # TODO: throw exception here
        if keys[0]:
            return {}

        d = {}

        # merge the data of all parsers first
        for p in self._parsers:
            for k, v in p.data().items():
                d[k] = v

        # if the second key is empty, '.' was passed and we return everything
        if not keys[1] and len(keys) is 2:
            return d

        # the first key is always in one of the parsers
        try:
            for k in keys[1:]:
                m = re.match(r'(?P<k>[^\[\]]*)(\[(?P<i>[\d\[\]]+)\])?$', k)
                if m.group('k') is not None:
                    d = d[m.group('k')]
                if m.group('i') is not None:
                    for i in m.group('i').split(']['):
                        # for dictionaries sort it alphabetically first (which makes it automatically a list of pairs)
                        if type(d) is dict:
                            d = sorted(d.items())
                        d = d[int(i)]

        # ignore key or type errors (when walking the parser data objects)
        except (TypeError, KeyError, IndexError):
            return {}

        return d
<|endoftext|>


<commit_before>from django.shortcuts import render, redirect
from django.conf import settings
from django.http import HttpResponse, Http404, HttpResponseBadRequest, HttpResponseForbidden
from django.views.generic import View
from django.views.decorators.clickjacking import xframe_options_exempt
from django.contrib.admin.views.decorators import staff_member_required
from django.core.exceptions import ObjectDoesNotExist
from mturk.queries import get_active_video_turk_task
from .models import *
from mturk.models import Task, FullVideoTask, SingleFrameTask
from .services import *
from datetime import datetime, timezone

import os
import json
import urllib.request
import urllib.parse
import markdown
import sys
import mturk.utils
from mturk.queries import get_active_video_turk_task
from .models import *
from .services import *

import logging
import ast

logger = logging.getLogger()

def home(request):
    need_annotating = Video.objects.filter(id__gt=0, verified=False)
    return render(request, 'video_list.html', context={
        'videos': need_annotating,
        'thumbnail': True,
        'test': settings.AWS_ID,
        'title': 'Videos'
    })

def verify_list(request):
    need_verification = Video.objects.filter(id__gt=0, verified=False).exclude(annotation='')[:250]
    return render(request, 'video_list.html', context={
        'videos': need_verification,
        'title': 'Videos to Verify'
    })

def verified_list(request):
    verified = Video.objects.filter(id__gt=0, verified=True).exclude(annotation='')[:100]
    return render(request, 'video_list.html', context={
        'videos': verified,
        'title': 'Verified Videos'
    })

def ready_to_pay(request):
    #tasks = FullVideoTask.objects.filter(paid = False, video__verified = True).exclude(hit_id = '')
    tasks = FullVideoTask.objects.all()#filter(paid = False, video__verified = True).exclude(hit_id = '')
    print("there are {} tasks".format(len(tasks)))
    return render(request, 'turk_ready_to_pay.html', context={
        'tasks': tasks,
    })

def next_unannotated(request, video_id):
    id = Video.objects.filter(id__gt=video_id, annotation='')[0].id
    return redirect('video', id)

# status of Not Published, Published, Awaiting Approval, Verified
# this is a bit convoluted as there's status stored on
# video (approved) as well as FullVideoTask (closed, paid, etc.)
def get_mturk_status(video, full_video_task):
    if video.verified:
        return "Verified"
    if full_video_task == None:
        if video.rejected == True:
            return "Rejected"
        elif video.annotation == '':
            return "Not Published"
        else:
            return "Awaiting Approval"
    if full_video_task.worker_id == '':
        return "Published"
    if full_video_task.worker_id != '':
        return "Awaiting Approval"

@xframe_options_exempt
def video(request, video_id):
    try:
        video = Video.objects.get(id=video_id)
        labels = Label.objects.all()
    except Video.DoesNotExist:
        raise Http404('No video with id "{}". Possible fixes: \n1) Download an up to date DB, see README. \n2) Add this video to the DB via /admin'.format(video_id))

    mturk_data = mturk.utils.authenticate_hit(request)
    if 'error' in mturk_data:
        return HttpResponseForbidden(mturk_data['error'])
    if not (mturk_data['authenticated'] or request.user.is_authenticated()):
        return redirect('/login/?next=' + request.path)

    start_time = float(request.GET['s']) if 's' in request.GET else None
    end_time = float(request.GET['e']) if 'e' in request.GET else None

    turk_task = get_active_video_turk_task(video.id)

    if turk_task != None:
        if turk_task.metrics != '':
            metricsDictr = ast.literal_eval(turk_task.metrics)
        else:
            metricsDictr = {}

        # Data for Javascript
        full_video_task_data = {
            'id': turk_task.id,
            'storedMetrics': metricsDictr,
            'bonus': float(turk_task.bonus),
            'bonusMessage': turk_task.message,
            'rejectionMessage': settings.MTURK_REJECTION_MESSAGE,
            'emailSubject': settings.MTURK_EMAIL_SUBJECT,
            'emailMessage': settings.MTURK_EMAIL_MESSAGE,
            'isComplete': turk_task.worker_id != ''
        }

        # Data for python templating
        if turk_task.last_email_sent_date != None:
            mturk_data['last_email_sent_date'] = turk_task.last_email_sent_date.strftime("%Y-%m-%d %H:%M")
    else:
        full_video_task_data = None

    mturk_data['status'] = get_mturk_status(video, turk_task)
    mturk_data['has_current_full_video_task'] = full_video_task_data != None

    video_data = json.dumps({
        'id': video.id,
        'location': video.url,
        'path': video.host,
        'is_image_sequence': True if video.image_list else False,
        'annotated': video.annotation != '',
        'verified': video.verified,
        'rejected': video.rejected,
        'start_time': start_time,
        'end_time' : end_time,
        'turk_task' : full_video_task_data
    })

    label_data = []
    video_labels = video.labels.all()
    if len(video_labels):
        for v_label in video_labels:
            label_data.append({'name': v_label.name, 'color': v_label.color})
    else:
        for l in labels:
            label_data.append({'name': l.name, 'color': l.color})

    help_content = ''
    if settings.HELP_URL and settings.HELP_USE_MARKDOWN:
        help_content = urllib.request.urlopen(settings.HELP_URL).read().decode('utf-8')
        help_content = markdown.markdown(help_content)

    response = render(request, 'video.html', context={
        'label_data': label_data,
        'video_data': video_data,
        'image_list': list(map(urllib.parse.quote, json.loads(video.image_list))) if video.image_list else 0,
        'image_list_path': urllib.parse.quote(video.host),
        'help_url': settings.HELP_URL,
        'help_embed': settings.HELP_EMBED,
        'mturk_data': mturk_data,
        'iframe_mode': mturk_data['authenticated'],
        'survey': False,
        'help_content': help_content
    })
    if not mturk_data['authenticated']:
        response['X-Frame-Options'] = 'SAMEORIGIN'
    return response

class AnnotationView(View):

    def get(self, request, video_id):
        video = Video.objects.get(id=video_id)
        return HttpResponse(video.annotation, content_type='application/json')

    def post(self, request, video_id):
        data = json.loads(request.body.decode('utf-8'))

        video = Video.objects.get(id=video_id)
        video.annotation = json.dumps(data['annotation'])
        video.save()

        hit_id = data.get('hitId', None)
        if hit_id != None:
            if not Task.valid_hit_id(hit_id):
                return HttpResponseForbidden('Not authenticated')
            else:
                try:
                    worker_id = data.get('workerId', '')
                    assignment_id = data.get('assignmentId', '')
                    task = Task.get_by_hit_id(hit_id)
                    task.complete(worker_id, assignment_id, data['metrics'])
                except ObjectDoesNotExist:
                    if not settings.DEBUG:
                        raise
        return HttpResponse('success')

class ReceiveCommand(View):

    def post(self, request, video_id):
        data = json.loads(request.body.decode('utf-8'))

        try:
            vid_id = int(video_id)
            command_type = data['type']

            if 'bonus' in data:
                bonus = data['bonus']
            message = data['message']
            reopen = data['reopen']
            delete_boxes = data['deleteBoxes']
            block_worker = data['blockWorker']
            updated_annotations = json.dumps(data['updatedAnnotations'])

            if command_type == "accept":
                accept_video(request, vid_id, bonus, message, reopen, delete_boxes, block_worker, updated_annotations)
            elif command_type == "reject":
                reject_video(request, vid_id, message, reopen, delete_boxes, block_worker, updated_annotations)
            elif command_type == "email":
                email_worker(request, vid_id, data['subject'], message)

            return HttpResponse(status=200)
        except Exception as e:
            logger.exception(e)
            response = HttpResponse(status=500)
            response['error-message'] = str(e)
            return response
<commit_msg>Replacing quote with unquote call (Facing bug with that) (#126)<commit_after>from django.shortcuts import render, redirect
from django.conf import settings
from django.http import HttpResponse, Http404, HttpResponseBadRequest, HttpResponseForbidden
from django.views.generic import View
from django.views.decorators.clickjacking import xframe_options_exempt
from django.contrib.admin.views.decorators import staff_member_required
from django.core.exceptions import ObjectDoesNotExist
from mturk.queries import get_active_video_turk_task
from .models import *
from mturk.models import Task, FullVideoTask, SingleFrameTask
from .services import *
from datetime import datetime, timezone

import os
import json
import urllib.request
import urllib.parse
import markdown
import sys
import mturk.utils
from mturk.queries import get_active_video_turk_task
from .models import *
from .services import *

import logging
import ast

logger = logging.getLogger()

def home(request):
    need_annotating = Video.objects.filter(id__gt=0, verified=False)
    return render(request, 'video_list.html', context={
        'videos': need_annotating,
        'thumbnail': True,
        'test': settings.AWS_ID,
        'title': 'Videos'
    })

def verify_list(request):
    need_verification = Video.objects.filter(id__gt=0, verified=False).exclude(annotation='')[:250]
    return render(request, 'video_list.html', context={
        'videos': need_verification,
        'title': 'Videos to Verify'
    })

def verified_list(request):
    verified = Video.objects.filter(id__gt=0, verified=True).exclude(annotation='')[:100]
    return render(request, 'video_list.html', context={
        'videos': verified,
        'title': 'Verified Videos'
    })

def ready_to_pay(request):
    #tasks = FullVideoTask.objects.filter(paid = False, video__verified = True).exclude(hit_id = '')
    tasks = FullVideoTask.objects.all()#filter(paid = False, video__verified = True).exclude(hit_id = '')
    print("there are {} tasks".format(len(tasks)))
    return render(request, 'turk_ready_to_pay.html', context={
        'tasks': tasks,
    })

def next_unannotated(request, video_id):
    id = Video.objects.filter(id__gt=video_id, annotation='')[0].id
    return redirect('video', id)

# status of Not Published, Published, Awaiting Approval, Verified
# this is a bit convoluted as there's status stored on
# video (approved) as well as FullVideoTask (closed, paid, etc.)
def get_mturk_status(video, full_video_task):
    if video.verified:
        return "Verified"
    if full_video_task == None:
        if video.rejected == True:
            return "Rejected"
        elif video.annotation == '':
            return "Not Published"
        else:
            return "Awaiting Approval"
    if full_video_task.worker_id == '':
        return "Published"
    if full_video_task.worker_id != '':
        return "Awaiting Approval"

@xframe_options_exempt
def video(request, video_id):
    try:
        video = Video.objects.get(id=video_id)
        labels = Label.objects.all()
    except Video.DoesNotExist:
        raise Http404('No video with id "{}". Possible fixes: \n1) Download an up to date DB, see README. \n2) Add this video to the DB via /admin'.format(video_id))

    mturk_data = mturk.utils.authenticate_hit(request)
    if 'error' in mturk_data:
        return HttpResponseForbidden(mturk_data['error'])
    if not (mturk_data['authenticated'] or request.user.is_authenticated()):
        return redirect('/login/?next=' + request.path)

    start_time = float(request.GET['s']) if 's' in request.GET else None
    end_time = float(request.GET['e']) if 'e' in request.GET else None

    turk_task = get_active_video_turk_task(video.id)

    if turk_task != None:
        if turk_task.metrics != '':
            metricsDictr = ast.literal_eval(turk_task.metrics)
        else:
            metricsDictr = {}

        # Data for Javascript
        full_video_task_data = {
            'id': turk_task.id,
            'storedMetrics': metricsDictr,
            'bonus': float(turk_task.bonus),
            'bonusMessage': turk_task.message,
            'rejectionMessage': settings.MTURK_REJECTION_MESSAGE,
            'emailSubject': settings.MTURK_EMAIL_SUBJECT,
            'emailMessage': settings.MTURK_EMAIL_MESSAGE,
            'isComplete': turk_task.worker_id != ''
        }

        # Data for python templating
        if turk_task.last_email_sent_date != None:
            mturk_data['last_email_sent_date'] = turk_task.last_email_sent_date.strftime("%Y-%m-%d %H:%M")
    else:
        full_video_task_data = None

    mturk_data['status'] = get_mturk_status(video, turk_task)
    mturk_data['has_current_full_video_task'] = full_video_task_data != None

    video_data = json.dumps({
        'id': video.id,
        'location': video.url,
        'path': video.host,
        'is_image_sequence': True if video.image_list else False,
        'annotated': video.annotation != '',
        'verified': video.verified,
        'rejected': video.rejected,
        'start_time': start_time,
        'end_time' : end_time,
        'turk_task' : full_video_task_data
    })

    label_data = []
    video_labels = video.labels.all()
    if len(video_labels):
        for v_label in video_labels:
            label_data.append({'name': v_label.name, 'color': v_label.color})
    else:
        for l in labels:
            label_data.append({'name': l.name, 'color': l.color})

    help_content = ''
    if settings.HELP_URL and settings.HELP_USE_MARKDOWN:
        help_content = urllib.request.urlopen(settings.HELP_URL).read().decode('utf-8')
        help_content = markdown.markdown(help_content)

    response = render(request, 'video.html', context={
        'label_data': label_data,
        'video_data': video_data,
        'image_list': list(map(urllib.parse.quote, json.loads(video.image_list))) if video.image_list else 0,
        'image_list_path': urllib.parse.quote(video.host, safe='/:'),
        'help_url': settings.HELP_URL,
        'help_embed': settings.HELP_EMBED,
        'mturk_data': mturk_data,
        'iframe_mode': mturk_data['authenticated'],
        'survey': False,
        'help_content': help_content
    })
    if not mturk_data['authenticated']:
        response['X-Frame-Options'] = 'SAMEORIGIN'
    return response

class AnnotationView(View):

    def get(self, request, video_id):
        video = Video.objects.get(id=video_id)
        return HttpResponse(video.annotation, content_type='application/json')

    def post(self, request, video_id):
        data = json.loads(request.body.decode('utf-8'))

        video = Video.objects.get(id=video_id)
        video.annotation = json.dumps(data['annotation'])
        video.save()

        hit_id = data.get('hitId', None)
        if hit_id != None:
            if not Task.valid_hit_id(hit_id):
                return HttpResponseForbidden('Not authenticated')
            else:
                try:
                    worker_id = data.get('workerId', '')
                    assignment_id = data.get('assignmentId', '')
                    task = Task.get_by_hit_id(hit_id)
                    task.complete(worker_id, assignment_id, data['metrics'])
                except ObjectDoesNotExist:
                    if not settings.DEBUG:
                        raise
        return HttpResponse('success')

class ReceiveCommand(View):

    def post(self, request, video_id):
        data = json.loads(request.body.decode('utf-8'))

        try:
            vid_id = int(video_id)
            command_type = data['type']

            if 'bonus' in data:
                bonus = data['bonus']
            message = data['message']
            reopen = data['reopen']
            delete_boxes = data['deleteBoxes']
            block_worker = data['blockWorker']
            updated_annotations = json.dumps(data['updatedAnnotations'])

            if command_type == "accept":
                accept_video(request, vid_id, bonus, message, reopen, delete_boxes, block_worker, updated_annotations)
            elif command_type == "reject":
                reject_video(request, vid_id, message, reopen, delete_boxes, block_worker, updated_annotations)
            elif command_type == "email":
                email_worker(request, vid_id, data['subject'], message)

            return HttpResponse(status=200)
        except Exception as e:
            logger.exception(e)
            response = HttpResponse(status=500)
            response['error-message'] = str(e)
            return response
<|endoftext|>


<commit_before>def determine_type(obj):
    """ Recursively queries a type and returns a map of the object
    with the named fields and their types.
    """
    if obj is None:
        return None
    elif isinstance(obj, str):
        return str
    elif isinstance(obj, int):
        return int
    elif isinstance(obj, float):
        return float
    elif isinstance(obj, list):
        return list(map(determine_type, obj))
    elif isinstance(obj, dict):
        return { k: determine_type(obj[k]) for k in obj.keys() }
    raise NotImplemented('Unknown type {0!r}'.format(obj))
<commit_msg>python2.6 doesn't support list comprehension<commit_after>def determine_type(obj):
    """ Recursively queries a type and returns a map of the object
    with the named fields and their types.
    """
    if obj is None:
        return None
    elif isinstance(obj, str):
        return str
    elif isinstance(obj, int):
        return int
    elif isinstance(obj, float):
        return float
    elif isinstance(obj, list):
        return list(map(determine_type, obj))
    elif isinstance(obj, dict):
        return dict(( (k, determine_type(obj[k]))
                      for k in obj.keys() ))
    raise NotImplemented('Unknown type {0!r}'.format(obj))
<|endoftext|>


<commit_before>import json
import time
from morph import flatten

def write_in_csv_format(iostream, frame):
    """
    Writes frame data and metadata into iostream in csv format.

    :param iostream: a CStringIO used to buffer the formatted features.
    :param frame: a BaseFrame object to be written into iostream
    :return: None
    """
    iostream.write('%s\t%s\t%s\n' %
                   ('metadata', json.dumps('metadata'),
                    json.dumps(frame.metadata, separators=(',', ':'))))
    for (key, val, feature_type) in frame.data:
        if not isinstance(val, dict):
            val = val._asdict()
        iostream.write('%s\t%s\t%s\n' % (
            feature_type, json.dumps(key),
            json.dumps(val, separators=(',', ':'))))

def write_in_json_format(iostream, frame):
    """
    Writes frame data and metadata into iostream in json format.

    :param iostream: a CStringIO used to buffer the formatted features.
    :param frame: a BaseFrame object to be written into iostream
    :return: None
    """
    iostream.write('%s\n' % json.dumps(frame.metadata))
    for (key, val, feature_type) in frame.data:
        if not isinstance(val, dict):
            val = val._asdict()
        val['feature_type'] = feature_type
        val['namespace'] = frame.metadata.get('namespace', '')
        iostream.write('%s\n' % json.dumps(val))

def write_in_graphite_format(iostream, frame):
    """
    Writes frame data and metadata into iostream in graphite format.

    :param iostream: a CStringIO used to buffer the formatted features.
    :param frame: a BaseFrame object to be written into iostream
    :return: None
    """
    namespace = frame.metadata.get('namespace', '')
    for (key, val, feature_type) in frame.data:
        if not isinstance(val, dict):
            val = val._asdict()
        write_feature_in_graphite_format(iostream, namespace,
                                         key, val, feature_type)

def write_feature_in_graphite_format(iostream, namespace,
                                     feature_key, feature_val,
                                     feature_type):
    """
    Write a feature in graphite format into iostream. The graphite format
    looks like this, one line per metric value:

        [namespace].[feature_key].[metric] [value] [timestamp]\r\n
        [namespace].[feature_key].[metric] [value] [timestamp]\r\n
        [namespace].[feature_key].[metric] [value] [timestamp]\r\n

    This function converts a feature into that string and writes it into
    the iostream.

    :param namespace: Frame namespace for this feature
    :param feature_type:
    :param feature_key:
    :param feature_val:
    :param iostream: a CStringIO used to buffer the formatted features.
    :return: None
    """
    timestamp = time.time()
    items = flatten(feature_val).items()
    if isinstance(namespace, dict):
        namespace = json.dumps(namespace)
    else:
        namespace = namespace.replace('/', '.')

    for (metric, value) in items:
        try:
            # Only emit values that we can cast as floats
            value = float(value)
        except (TypeError, ValueError):
            continue

        metric = metric.replace('(', '_').replace(')', '')
        metric = metric.replace(' ', '_').replace('-', '_')
        metric = metric.replace('/', '_').replace('\\', '_')

        feature_key = feature_key.replace('_', '-')
        if 'cpu' in feature_key or 'memory' in feature_key:
            metric = metric.replace('_', '-')
        if 'if' in metric:
            metric = metric.replace('_tx', '.tx')
            metric = metric.replace('_rx', '.rx')
        if feature_key == 'load':
            feature_key = 'load.load'
        feature_key = feature_key.replace('/', '$')

        tmp_message = '%s.%s.%s %f %d\r\n' % (namespace, feature_key,
                                              metric, value, timestamp)
        iostream.write(tmp_message)
<commit_msg>fix time bug in format-to-graphite<commit_after>import json
import time
from morph import flatten

def write_in_csv_format(iostream, frame):
    """
    Writes frame data and metadata into iostream in csv format.

    :param iostream: a CStringIO used to buffer the formatted features.
    :param frame: a BaseFrame object to be written into iostream
    :return: None
    """
    iostream.write('%s\t%s\t%s\n' %
                   ('metadata', json.dumps('metadata'),
                    json.dumps(frame.metadata, separators=(',', ':'))))
    for (key, val, feature_type) in frame.data:
        if not isinstance(val, dict):
            val = val._asdict()
        iostream.write('%s\t%s\t%s\n' % (
            feature_type, json.dumps(key),
            json.dumps(val, separators=(',', ':'))))

def write_in_json_format(iostream, frame):
    """
    Writes frame data and metadata into iostream in json format.

    :param iostream: a CStringIO used to buffer the formatted features.
    :param frame: a BaseFrame object to be written into iostream
    :return: None
    """
    iostream.write('%s\n' % json.dumps(frame.metadata))
    for (key, val, feature_type) in frame.data:
        if not isinstance(val, dict):
            val = val._asdict()
        val['feature_type'] = feature_type
        val['namespace'] = frame.metadata.get('namespace', '')
        iostream.write('%s\n' % json.dumps(val))

def write_in_graphite_format(iostream, frame):
    """
    Writes frame data and metadata into iostream in graphite format.

    :param iostream: a CStringIO used to buffer the formatted features.
    :param frame: a BaseFrame object to be written into iostream
    :return: None
    """
    namespace = frame.metadata.get('namespace', '')
    timestamp = frame.metadata.get('timestamp', '')
    for (key, val, feature_type) in frame.data:
        if not isinstance(val, dict):
            val = val._asdict()
        write_feature_in_graphite_format(iostream, namespace, timestamp,
                                         key, val, feature_type)

def write_feature_in_graphite_format(iostream, namespace, timestamp,
                                     feature_key, feature_val,
                                     feature_type):
    """
    Write a feature in graphite format into iostream. The graphite format
    looks like this, one line per metric value:

        [namespace].[feature_key].[metric] [value] [timestamp]\r\n
        [namespace].[feature_key].[metric] [value] [timestamp]\r\n
        [namespace].[feature_key].[metric] [value] [timestamp]\r\n

    This function converts a feature into that string and writes it into
    the iostream.

    :param namespace: Frame namespace for this feature
    :param timestamp: From frame metadata, fmt: %Y-%m-%dT%H:%M:%S%z
    :param feature_type:
    :param feature_key:
    :param feature_val:
    :param iostream: a CStringIO used to buffer the formatted features.
    :return: None
    """
    # to convert 2017-02-07T13:20:15-0500 to 1486491615 (=epoch)
    # for python >=3.2, following works
    # time.strptime(timestamp,'%Y-%m-%dT%H:%M:%S%z'),
    # but previous pyhon versions don't respect %z timezone info,
    # so skipping time zone conversion assuming
    # timezone during str timestamp creation in metadata is same for reverse

    timestamp = time.mktime(time.strptime(timestamp[:-5], '%Y-%m-%dT%H:%M:%S'))

    items = flatten(feature_val).items()
    if isinstance(namespace, dict):
        namespace = json.dumps(namespace)
    else:
        namespace = namespace.replace('/', '.')

    for (metric, value) in items:
        try:
            # Only emit values that we can cast as floats
            value = float(value)
        except (TypeError, ValueError):
            continue

        metric = metric.replace('(', '_').replace(')', '')
        metric = metric.replace(' ', '_').replace('-', '_')
        metric = metric.replace('/', '_').replace('\\', '_')

        feature_key = feature_key.replace('_', '-')
        if 'cpu' in feature_key or 'memory' in feature_key:
            metric = metric.replace('_', '-')
        if 'if' in metric:
            metric = metric.replace('_tx', '.tx')
            metric = metric.replace('_rx', '.rx')
        if feature_key == 'load':
            feature_key = 'load.load'
        feature_key = feature_key.replace('/', '$')

        tmp_message = '%s.%s.%s %f %d\r\n' % (namespace, feature_key,
                                              metric, value, timestamp)
        iostream.write(tmp_message)
<|endoftext|>