<commit_before>""" Things one can do relating to a public records request. """

from public_records_portal import app, models, website_copy
from models import *
import os
import tempfile
import scribd # Used for uploading documents, could use another service
import time
import requests as seaturtle # HTTP requests library, renaming because it's confusing otherwise
import json
from flask import Flask, render_template, request, flash, redirect, url_for
from flask.ext.sqlalchemy import SQLAlchemy, sqlalchemy
from sqlalchemy.exc import IntegrityError, InvalidRequestError
from werkzeug import secure_filename
import sendgrid

mail = sendgrid.Sendgrid(app.config['MAIL_USERNAME'], app.config['MAIL_PASSWORD'], secure = True)
if app.config['ENVIRONMENT'] == "STAGING":
	app_url = app.config['APPLICATION_URL']
elif app.config['ENVIRONMENT'] == 'PRODUCTION':
	app_url = None
else:
	app_url = "http://127.0.0.1:8000/"

# Define the local temporary folder where uploads will go
if app.config['ENVIRONMENT'] == "PRODUCTION":
	UPLOAD_FOLDER = None # To do
elif app.config['ENVIRONMENT'] == "STAGING":
	UPLOAD_FOLDER = "/app/tmp"
else:
	UPLOAD_FOLDER = "%s/uploads" % os.getcwd()
ALLOWED_EXTENSIONS = set(['txt', 'pdf', 'doc', 'ps', 'rtf', 'epub', 'key', 'odt', 'odp', 'ods', 'odg', 'odf', 'sxw', 'sxc', 'sxi', 'sxd', 'ppt', 'pps', 'xls', 'zip', 'docx', 'pptx', 'ppsx', 'xlsx', 'tif', 'tiff'])

def get_resource(resource, resource_id, url = None):
	url = app_url
	headers = {'Content-Type': 'application/json'}
	r = seaturtle.get("%s/api/%s/%s" %(url, resource, resource_id), headers=headers)
	if r:
		return r.json()
	return None

def get_resource_filter(resource, filters, url = None):
	url = app_url
	headers = {'Content-Type': 'application/json'}
	params = dict(q=json.dumps(dict(filters=filters)))
	r = seaturtle.get("%s/api/%s" %(url, resource), params=params, headers = headers)
	if r:
		return r.json()
	return None

def get_resources(resource, url = None):
	url = app_url
	headers = {'content-type': 'application/json; charset=utf-8'}
	r = seaturtle.get("%s/api/%s" %(url, resource), headers=headers)
	if r:
		return r.json()
	return None

def add_resource(resource, request_body, current_user_id = None):
	fields = request_body.form
	if "note" in resource:
		add_note(fields['request_id'], fields['note_text'], current_user_id)
	elif "record" in resource:
		upload_record(fields['request_id'], request.files['record'], fields['record_description'])
	elif "link" in resource:
		add_link(fields['request_id'], fields['link_url'], fields['link_description'])
	elif "qa" in resource:
		ask_a_question(fields['request_id'], fields['user_id'], fields['question_text'])
	else:
		return False
	return True

def create_resource(resource, payload, url = None):
	url = app_url
	r = seaturtle.post("%s/api/%s" %(url, resource), data=json.dumps(payload))
	if r:
		return r.json()
	return None

def delete_resource(resource, resource_id, url = None):
	url = app_url
	r = seaturtle.delete("%s/api/%s/%s" %(url, resource, resource_id))
	if r.status_code == '204':
		return True
	return False

def put_resource(resource, payload, resource_id, url = None):
	url = app_url
	r = seaturtle.put("%s/api/%s/%s" %(url, resource, resource_id), data=json.dumps(payload))
	if r.status_code == '201':
		return True
	return False

def update_resource(resource, request_body):
	fields = request_body.form
	if "qa" in resource:
		answer_a_question(fields['qa_id'], fields['answer_text'], fields['user_id'])
		return True
	else:
		return False

def add_note(request_id, text, user_id):
	note = Note(request_id = request_id, text = text, user_id = user_id)
	db.session.add(note)
	db.session.commit()
	change_request_status(request_id, "A response has been added.")
	send_prr_email(request_id = request_id, notification_type = "Response added", requester_id = get_requester(request_id))

def upload_record(request_id, file, description):
	doc_id, filename = upload_file(file)
	req = get_resource("request", request_id)
	if str(doc_id).isdigit():
		record = Record(doc_id = doc_id, request_id = request_id, owner_id = req['current_owner'], description = description)
		db.session.add(record)
		db.session.commit()
		record.filename = filename
		record.url = app.config['HOST_URL'] + doc_id
		db.session.commit()
		change_request_status(request_id, "A response has been added.")
		send_prr_email(request_id = request_id, notification_type = "Response added", requester_id = get_requester(request_id))
	else:
		return "Not an allowed doc type"

def add_link(request_id, url, description):
	req = get_resource("request", request_id)
	record = Record(url = url, request_id = request_id, owner_id = req['current_owner'], description = description)
	db.session.add(record)
	db.session.commit()
	change_request_status(request_id, "A response has been added.")
	send_prr_email(request_id = request_id, notification_type = "Response added", requester_id = get_requester(request_id))

def make_request(text, email = None, assigned_to_name = None, assigned_to_email = None, assigned_to_reason = None, user_id = None):
	""" Make the request. At minimum you need to communicate which record(s) you want, probably with some text."""
	resource = get_resource_filter("request", [dict(name='text', op='eq', val=text)])
	if not resource or not resource['objects']:
		payload = dict(text=text)
		if user_id:
			payload = dict(text=text, creator_id = user_id)
		req = create_resource("request", payload)
		past_owner_id, owner_id = assign_owner(request_id = req['id'], reason = assigned_to_reason, email = assigned_to_email, alias = assigned_to_name)
		if email: # If the user provided an e-mail address, add them as a subscriber to the request.
			user = create_or_return_user(email = email)
			subscriber = Subscriber(request_id = req['id'], user_id = user['id'])
			subscriber.creator = True
			db.session.add(subscriber)
			db.session.commit()
		open_request(req['id'])
		return req['id'], True
	return resource['objects'][0]['id'], False

def ask_a_question(request_id, user_id, question):
	""" City staff can ask a question about a request they are confused about."""
	qa = QA(request_id = request_id, question = question)
	qa.owner_id = user_id
	db.session.add(qa)
	db.session.commit()
	change_request_status(request_id, "Pending")
	send_prr_email(request_id = qa.request_id, notification_type = "Question asked", requester_id = get_requester(request_id))
	return qa.id

def answer_a_question(qa_id, answer, subscriber_id = None):
	""" A requester can answer a question city staff asked them about their request."""
	qa = get_resource("qa", qa_id)
	put_resource("qa", dict(subscriber_id = subscriber_id, answer = answer),int(qa_id))
	change_request_status(qa['request_id'], "Pending")
	# change_request_status(request_id, "%s needs to take action." %) # Pass the buck to the current owner
	req = get_resource("request", qa['request_id'])
	send_prr_email(request_id = qa['request_id'], notification_type = "Question answered", owner_id = req['current_owner'])

def create_or_return_user(email, alias = None):
	resource = get_resource_filter("user", [dict(name='email', op='eq', val=email)])
	if not resource['objects']:
		user = create_resource("user", dict(email = email, alias = alias))
		return user
	return resource['objects'][0]

def open_request(request_id):
	change_request_status(request_id, "Open")

def close_request(request_id, reason = ""):
	change_request_status(request_id, "Closed. %s" %reason)
	send_prr_email(request_id = request_id, notification_type = "Request closed", requester_id = get_requester(request_id))

def get_subscribers(request_id):
	req = get_resource("request", request_id)
	return req['subscribers']

def get_requester(request_id):
	subscribers = get_subscribers(request_id)
	for subscriber in subscribers:
		return subscriber['id'] # Return first one for now
		# if 'creator' in subscriber:
		# 	if subscriber['creator'] == True:
		# 		return subscriber.id

def assign_owner(request_id, reason, email = None, alias = None):
	""" Called any time a new owner is assigned. This will overwrite the current owner."""
	user = create_or_return_user(email = email, alias = alias)
	req = get_resource("request", request_id)
	current_owner_id = None
	if not req['current_owner']:
		current_owner_id = req['current_owner']
	owner = None
	owner_resource = get_resource_filter("owner", [dict(name='request_id', op='eq', val=int(request_id)), dict(name='user_id', op='eq', val=int(user['id']))])
	if owner_resource['objects']:
		owner = owner_resource['objects']['0']
	if current_owner_id and owner:
		if current_owner_id == owner['id']:
			return None, None
	new_owner = create_resource("owner", dict(request_id = request_id, user_id = user['id'], reason = reason))
	if current_owner_id:
		past_owner_id = current_owner_id
	else:
		past_owner_id = None
	put_resource("request", dict(current_owner = new_owner['id']) ,int(request_id))
	change_request_status(request_id, "Pending")
	send_prr_email(request_id = request_id, notification_type = "Request assigned", owner_id = new_owner['id'])
	return past_owner_id, new_owner['id']

def change_request_status(request_id, status):
	return put_resource("request", dict(status = status, status_updated = datetime.now().isoformat()),int(request_id))

def progress(bytes_sent, bytes_total):
    print("%s of %s (%s%%)" % (bytes_sent, bytes_total, bytes_sent*100/bytes_total))

def upload(filepath, API_KEY, API_SECRET):
    # Configure the Scribd API.
    print "This is the filepath passed to upload %s" %(filepath)
    scribd.config(API_KEY, API_SECRET)
    doc_id = None
    try:
        # Upload the document from a file.
        doc = scribd.api_user.upload(
            open(filepath,'rb'),
            progress_callback=progress,
            req_buffer = tempfile.TemporaryFile()
            )
        # Poll API until conversion is complete.
        while doc.get_conversion_status() != 'DONE':
            # Sleep to prevent a runaway loop that will block the script.
            time.sleep(2)
        doc_id = doc.id
        return doc_id
    except scribd.ResponseError, err:
        print 'Scribd failed: code=%d, error=%s' % (err.errno, err.strerror)
        return err.strerror

def make_public(doc_id, API_KEY, API_SECRET):
    scribd.config(API_KEY, API_SECRET)
    doc = scribd.api_user.get(doc_id)
    doc.access = 'public'
    doc.save()

def make_private(doc_id, API_KEY, API_SECRET):
    scribd.config(API_KEY, API_SECRET)
    doc = scribd.api_user.get(doc_id)
    doc.access = 'private'
    doc.save()

def allowed_file(filename):
	return '.' in filename and \
		filename.rsplit('.', 1)[1] in ALLOWED_EXTENSIONS

def email_validation(email):
	name, domain = email.split("@")
	if domain == "oakland.net" or domain == "oaklandnet.com" or domain == "codeforamerica.org":
		return True
	return False

def upload_file(file):
# Uploads file to scribd.com and returns doc ID. File can be accessed at scribd.com/doc/id
	if file and allowed_file(file.filename):
		filename = secure_filename(file.filename)
		filepath = os.path.join(UPLOAD_FOLDER, filename)
		print "This is the filepath: %s" % (filepath)
		file.save(filepath)
		doc_id = upload(filepath, app.config['SCRIBD_API_KEY'], app.config['SCRIBD_API_SECRET'])
		return doc_id, filename
	return None, None

def send_prr_email(request_id, notification_type, requester_id = None, owner_id = None):
	email_json = open(os.path.join(app.root_path, 'emails.json'))
	json_data = json.load(email_json)
	email_subject = "Public Records Request %s: %s" %(request_id, json_data[notification_type])
	page = None
	uid = None
	if owner_id:
		page = "%scity/request/%s" %(app_url,request_id)
		owner = get_resource("owner", owner_id)
		uid = owner['user_id']
	if requester_id:
		page = "%srequest/%s" %(app_url,request_id)
		requester = get_resource("subscriber", requester_id)
		uid = requester['user_id']
	email_address = get_user_email(uid)
	if app.config['ENVIRONMENT'] == "PRODUCTION":
		print "%s to %s with subject %s" % (render_template("generic_email.html", page = page), email_address, email_subject)
	else:
		send_email(render_template("generic_email.html", page = page), email_address, email_subject)

def get_user_email(uid):
	user = get_resource("user", uid)
	return user['email']

def send_email(body, recipient, subject):
	sender = app.config['DEFAULT_MAIL_SENDER']
	plaintext = ""
	html = body
	message = sendgrid.Message(sender, subject, plaintext, html)
	message.add_to(recipient)
	message.add_bcc(sender)
	mail.web.send(message)

<commit_msg>UPLOAD_FOLDER for heroku update<commit_after>""" Things one can do relating to a public records request. """

from public_records_portal import app, models, website_copy
from models import *
import os
import tempfile
import scribd # Used for uploading documents, could use another service
import time
import requests as seaturtle # HTTP requests library, renaming because it's confusing otherwise
import json
from flask import Flask, render_template, request, flash, redirect, url_for
from flask.ext.sqlalchemy import SQLAlchemy, sqlalchemy
from sqlalchemy.exc import IntegrityError, InvalidRequestError
from werkzeug import secure_filename
import sendgrid

mail = sendgrid.Sendgrid(app.config['MAIL_USERNAME'], app.config['MAIL_PASSWORD'], secure = True)
if app.config['ENVIRONMENT'] == "STAGING":
	app_url = app.config['APPLICATION_URL']
elif app.config['ENVIRONMENT'] == 'PRODUCTION':
	app_url = None
else:
	app_url = "http://127.0.0.1:8000/"

# Define the local temporary folder where uploads will go
if app.config['ENVIRONMENT'] == "PRODUCTION":
	UPLOAD_FOLDER = None # To do
elif app.config['ENVIRONMENT'] == "STAGING":
	UPLOAD_FOLDER = "/app"
else:
	UPLOAD_FOLDER = "%s/uploads" % os.getcwd()
ALLOWED_EXTENSIONS = set(['txt', 'pdf', 'doc', 'ps', 'rtf', 'epub', 'key', 'odt', 'odp', 'ods', 'odg', 'odf', 'sxw', 'sxc', 'sxi', 'sxd', 'ppt', 'pps', 'xls', 'zip', 'docx', 'pptx', 'ppsx', 'xlsx', 'tif', 'tiff'])

def get_resource(resource, resource_id, url = None):
	url = app_url
	headers = {'Content-Type': 'application/json'}
	r = seaturtle.get("%s/api/%s/%s" %(url, resource, resource_id), headers=headers)
	if r:
		return r.json()
	return None

def get_resource_filter(resource, filters, url = None):
	url = app_url
	headers = {'Content-Type': 'application/json'}
	params = dict(q=json.dumps(dict(filters=filters)))
	r = seaturtle.get("%s/api/%s" %(url, resource), params=params, headers = headers)
	if r:
		return r.json()
	return None

def get_resources(resource, url = None):
	url = app_url
	headers = {'content-type': 'application/json; charset=utf-8'}
	r = seaturtle.get("%s/api/%s" %(url, resource), headers=headers)
	if r:
		return r.json()
	return None

def add_resource(resource, request_body, current_user_id = None):
	fields = request_body.form
	if "note" in resource:
		add_note(fields['request_id'], fields['note_text'], current_user_id)
	elif "record" in resource:
		upload_record(fields['request_id'], request.files['record'], fields['record_description'])
	elif "link" in resource:
		add_link(fields['request_id'], fields['link_url'], fields['link_description'])
	elif "qa" in resource:
		ask_a_question(fields['request_id'], fields['user_id'], fields['question_text'])
	else:
		return False
	return True

def create_resource(resource, payload, url = None):
	url = app_url
	r = seaturtle.post("%s/api/%s" %(url, resource), data=json.dumps(payload))
	if r:
		return r.json()
	return None

def delete_resource(resource, resource_id, url = None):
	url = app_url
	r = seaturtle.delete("%s/api/%s/%s" %(url, resource, resource_id))
	if r.status_code == '204':
		return True
	return False

def put_resource(resource, payload, resource_id, url = None):
	url = app_url
	r = seaturtle.put("%s/api/%s/%s" %(url, resource, resource_id), data=json.dumps(payload))
	if r.status_code == '201':
		return True
	return False

def update_resource(resource, request_body):
	fields = request_body.form
	if "qa" in resource:
		answer_a_question(fields['qa_id'], fields['answer_text'], fields['user_id'])
		return True
	else:
		return False

def add_note(request_id, text, user_id):
	note = Note(request_id = request_id, text = text, user_id = user_id)
	db.session.add(note)
	db.session.commit()
	change_request_status(request_id, "A response has been added.")
	send_prr_email(request_id = request_id, notification_type = "Response added", requester_id = get_requester(request_id))

def upload_record(request_id, file, description):
	doc_id, filename = upload_file(file)
	req = get_resource("request", request_id)
	if str(doc_id).isdigit():
		record = Record(doc_id = doc_id, request_id = request_id, owner_id = req['current_owner'], description = description)
		db.session.add(record)
		db.session.commit()
		record.filename = filename
		record.url = app.config['HOST_URL'] + doc_id
		db.session.commit()
		change_request_status(request_id, "A response has been added.")
		send_prr_email(request_id = request_id, notification_type = "Response added", requester_id = get_requester(request_id))
	else:
		return "Not an allowed doc type"

def add_link(request_id, url, description):
	req = get_resource("request", request_id)
	record = Record(url = url, request_id = request_id, owner_id = req['current_owner'], description = description)
	db.session.add(record)
	db.session.commit()
	change_request_status(request_id, "A response has been added.")
	send_prr_email(request_id = request_id, notification_type = "Response added", requester_id = get_requester(request_id))

def make_request(text, email = None, assigned_to_name = None, assigned_to_email = None, assigned_to_reason = None, user_id = None):
	""" Make the request. At minimum you need to communicate which record(s) you want, probably with some text."""
	resource = get_resource_filter("request", [dict(name='text', op='eq', val=text)])
	if not resource or not resource['objects']:
		payload = dict(text=text)
		if user_id:
			payload = dict(text=text, creator_id = user_id)
		req = create_resource("request", payload)
		past_owner_id, owner_id = assign_owner(request_id = req['id'], reason = assigned_to_reason, email = assigned_to_email, alias = assigned_to_name)
		if email: # If the user provided an e-mail address, add them as a subscriber to the request.
			user = create_or_return_user(email = email)
			subscriber = Subscriber(request_id = req['id'], user_id = user['id'])
			subscriber.creator = True
			db.session.add(subscriber)
			db.session.commit()
		open_request(req['id'])
		return req['id'], True
	return resource['objects'][0]['id'], False

def ask_a_question(request_id, user_id, question):
	""" City staff can ask a question about a request they are confused about."""
	qa = QA(request_id = request_id, question = question)
	qa.owner_id = user_id
	db.session.add(qa)
	db.session.commit()
	change_request_status(request_id, "Pending")
	send_prr_email(request_id = qa.request_id, notification_type = "Question asked", requester_id = get_requester(request_id))
	return qa.id

def answer_a_question(qa_id, answer, subscriber_id = None):
	""" A requester can answer a question city staff asked them about their request."""
	qa = get_resource("qa", qa_id)
	put_resource("qa", dict(subscriber_id = subscriber_id, answer = answer),int(qa_id))
	change_request_status(qa['request_id'], "Pending")
	# change_request_status(request_id, "%s needs to take action." %) # Pass the buck to the current owner
	req = get_resource("request", qa['request_id'])
	send_prr_email(request_id = qa['request_id'], notification_type = "Question answered", owner_id = req['current_owner'])

def create_or_return_user(email, alias = None):
	resource = get_resource_filter("user", [dict(name='email', op='eq', val=email)])
	if not resource['objects']:
		user = create_resource("user", dict(email = email, alias = alias))
		return user
	return resource['objects'][0]

def open_request(request_id):
	change_request_status(request_id, "Open")

def close_request(request_id, reason = ""):
	change_request_status(request_id, "Closed. %s" %reason)
	send_prr_email(request_id = request_id, notification_type = "Request closed", requester_id = get_requester(request_id))

def get_subscribers(request_id):
	req = get_resource("request", request_id)
	return req['subscribers']

def get_requester(request_id):
	subscribers = get_subscribers(request_id)
	for subscriber in subscribers:
		return subscriber['id'] # Return first one for now
		# if 'creator' in subscriber:
		# 	if subscriber['creator'] == True:
		# 		return subscriber.id

def assign_owner(request_id, reason, email = None, alias = None):
	""" Called any time a new owner is assigned. This will overwrite the current owner."""
	user = create_or_return_user(email = email, alias = alias)
	req = get_resource("request", request_id)
	current_owner_id = None
	if not req['current_owner']:
		current_owner_id = req['current_owner']
	owner = None
	owner_resource = get_resource_filter("owner", [dict(name='request_id', op='eq', val=int(request_id)), dict(name='user_id', op='eq', val=int(user['id']))])
	if owner_resource['objects']:
		owner = owner_resource['objects']['0']
	if current_owner_id and owner:
		if current_owner_id == owner['id']:
			return None, None
	new_owner = create_resource("owner", dict(request_id = request_id, user_id = user['id'], reason = reason))
	if current_owner_id:
		past_owner_id = current_owner_id
	else:
		past_owner_id = None
	put_resource("request", dict(current_owner = new_owner['id']) ,int(request_id))
	change_request_status(request_id, "Pending")
	send_prr_email(request_id = request_id, notification_type = "Request assigned", owner_id = new_owner['id'])
	return past_owner_id, new_owner['id']

def change_request_status(request_id, status):
	return put_resource("request", dict(status = status, status_updated = datetime.now().isoformat()),int(request_id))

def progress(bytes_sent, bytes_total):
    print("%s of %s (%s%%)" % (bytes_sent, bytes_total, bytes_sent*100/bytes_total))

def upload(filepath, API_KEY, API_SECRET):
    # Configure the Scribd API.
    print "This is the filepath passed to upload %s" %(filepath)
    scribd.config(API_KEY, API_SECRET)
    doc_id = None
    try:
        # Upload the document from a file.
        doc = scribd.api_user.upload(
            open(filepath,'rb'),
            progress_callback=progress,
            req_buffer = tempfile.TemporaryFile()
            )
        # Poll API until conversion is complete.
        while doc.get_conversion_status() != 'DONE':
            # Sleep to prevent a runaway loop that will block the script.
            time.sleep(2)
        doc_id = doc.id
        return doc_id
    except scribd.ResponseError, err:
        print 'Scribd failed: code=%d, error=%s' % (err.errno, err.strerror)
        return err.strerror

def make_public(doc_id, API_KEY, API_SECRET):
    scribd.config(API_KEY, API_SECRET)
    doc = scribd.api_user.get(doc_id)
    doc.access = 'public'
    doc.save()

def make_private(doc_id, API_KEY, API_SECRET):
    scribd.config(API_KEY, API_SECRET)
    doc = scribd.api_user.get(doc_id)
    doc.access = 'private'
    doc.save()

def allowed_file(filename):
	return '.' in filename and \
		filename.rsplit('.', 1)[1] in ALLOWED_EXTENSIONS

def email_validation(email):
	name, domain = email.split("@")
	if domain == "oakland.net" or domain == "oaklandnet.com" or domain == "codeforamerica.org":
		return True
	return False

def upload_file(file):
# Uploads file to scribd.com and returns doc ID. File can be accessed at scribd.com/doc/id
	if file and allowed_file(file.filename):
		filename = secure_filename(file.filename)
		filepath = os.path.join(UPLOAD_FOLDER, filename)
		print "This is the filepath: %s" % (filepath)
		file.save(filepath)
		doc_id = upload(filepath, app.config['SCRIBD_API_KEY'], app.config['SCRIBD_API_SECRET'])
		return doc_id, filename
	return None, None

def send_prr_email(request_id, notification_type, requester_id = None, owner_id = None):
	email_json = open(os.path.join(app.root_path, 'emails.json'))
	json_data = json.load(email_json)
	email_subject = "Public Records Request %s: %s" %(request_id, json_data[notification_type])
	page = None
	uid = None
	if owner_id:
		page = "%scity/request/%s" %(app_url,request_id)
		owner = get_resource("owner", owner_id)
		uid = owner['user_id']
	if requester_id:
		page = "%srequest/%s" %(app_url,request_id)
		requester = get_resource("subscriber", requester_id)
		uid = requester['user_id']
	email_address = get_user_email(uid)
	if app.config['ENVIRONMENT'] == "PRODUCTION":
		print "%s to %s with subject %s" % (render_template("generic_email.html", page = page), email_address, email_subject)
	else:
		send_email(render_template("generic_email.html", page = page), email_address, email_subject)

def get_user_email(uid):
	user = get_resource("user", uid)
	return user['email']

def send_email(body, recipient, subject):
	sender = app.config['DEFAULT_MAIL_SENDER']
	plaintext = ""
	html = body
	message = sendgrid.Message(sender, subject, plaintext, html)
	message.add_to(recipient)
	message.add_bcc(sender)
	mail.web.send(message)

<|endoftext|>


<commit_before>import hashlib
import os
import shutil

from django.conf import settings
from django.core import management
from django.test import TestCase
from django.test.client import Client

from server.models import Data, Case, Storage, Processor, GenUser, iterate_schema
from server.tasks import manager
from ..unit.utils import create_test_case, clear_all

class BaseProcessorTestCase(TestCase):
    @classmethod
    def setUpClass(cls):
        clear_all()

        super(BaseProcessorTestCase, cls).setUpClass()

        cls.user = GenUser.objects.create_superuser(email="admin@genialis.com")
        management.call_command('register')

    def setUp(self):
        super(BaseProcessorTestCase, self).setUp()

        self.current_path = os.path.dirname(os.path.abspath(__file__))
        self.case = create_test_case(self.user.pk)['c1']

    def tearDown(self):
        super(BaseProcessorTestCase, self).tearDown()

        # Delete Data objects and their files
        for d in Data.objects.all():
            data_dir = os.path.join(settings.DATAFS['data_path'], str(d.pk))
            shutil.rmtree(data_dir, ignore_errors=True)
            d.delete()

    def get_field(self, obj, path):
        for p in path.split('.'):
            obj = obj[p]
        return obj

    def run_processor(self, processor_name, input_):
        p = Processor.objects.get(name=processor_name)

        for field_schema, fields in iterate_schema(input_, p['input_schema']):
            # copy referenced files to upload dir
            if field_schema['type'] == "basic:file:":
                old_path = os.path.join(self.current_path, 'inputs', fields[field_schema['name']])
                shutil.copy2(old_path, settings.FILE_UPLOAD_TEMP_DIR)
                file_name = os.path.basename(fields[field_schema['name']])
                fields[field_schema['name']] = {
                    'file': file_name,
                    'file_temp': file_name,
                    'is_remote': False,
                }

            # convert primary keys to strings
            if field_schema['type'].startswith('data:'):
                fields[field_schema['name']] = str(fields[field_schema['name']])
            if field_schema['type'].startswith('list:data:'):
                fields[field_schema['name']] = [str(obj) for obj in fields[field_schema['name']]]

            # fill field with default values if empty
            if 'default' in field_schema and field_schema['name'] not in fields:
                fields[field_schema['name']] = field_schema['default']

        d = Data(
            input=input_,
            author_id=self.user.pk,
            processor_name=processor_name,
            case_ids=[self.case.pk],
        )
        d.save()
        manager(run_sync=True)

        return Data.objects.get(pk=d.pk)

    def assertDone(self, obj):
        self.assertEqual(obj.status, 'done')

    def assertFields(self, obj, path, value):
        field = self.get_field(obj['output'], path)
        self.assertEqual(field, str(value))

    def assertFiles(self, obj, field_path, fn):
        field = self.get_field(obj['output'], field_path)
        output = os.path.join(settings.DATAFS['data_path'], str(obj.pk), field['file'])
        output_hash = hashlib.sha256(open(output).read()).hexdigest()

        wanted = os.path.join(self.current_path, 'outputs', fn)
        wanted_hash = hashlib.sha256(open(wanted).read()).hexdigest()

        self.assertEqual(wanted_hash, output_hash)

    def assertJSON(self, storage_id, field_path, fn):
        storage = Storage.objects.get(pk=storage_id)

        field = self.get_field(storage['json'], field_path)
        field_hash = hashlib.sha256(field).hexdigest()

        wanted = os.path.join(self.current_path, 'outputs', fn)
        wanted_hash = hashlib.sha256(open(wanted).read()).hexdigest()

        self.assertEqual(wanted_hash, field_hash)
<commit_msg>Create output files if don't exist.<commit_after>import hashlib
import os
import shutil

from django.conf import settings
from django.core import management
from django.test import TestCase

from server.models import Data, Storage, Processor, GenUser, iterate_schema
from server.tasks import manager
from ..unit.utils import create_test_case, clear_all

class BaseProcessorTestCase(TestCase):
    @classmethod
    def setUpClass(cls):
        clear_all()

        super(BaseProcessorTestCase, cls).setUpClass()

        cls.user = GenUser.objects.create_superuser(email="admin@genialis.com")
        management.call_command('register')

        cls.created_files = []

    @classmethod
    def tearDownClass(cls):
        super(BaseProcessorTestCase, cls).tearDownClass()

        if len(cls.created_files):
            print "#" * 80
            print "WARNING: Next files were created: {}".format(', '.join(cls.created_files))
            print "#" * 80

    def setUp(self):
        super(BaseProcessorTestCase, self).setUp()

        self.current_path = os.path.dirname(os.path.abspath(__file__))
        self.case = create_test_case(self.user.pk)['c1']

    def tearDown(self):
        super(BaseProcessorTestCase, self).tearDown()

        # Delete Data objects and their files
        for d in Data.objects.all():
            data_dir = os.path.join(settings.DATAFS['data_path'], str(d.pk))
            shutil.rmtree(data_dir, ignore_errors=True)
            d.delete()

    def get_field(self, obj, path):
        for p in path.split('.'):
            obj = obj[p]
        return obj

    def run_processor(self, processor_name, input_):
        p = Processor.objects.get(name=processor_name)

        for field_schema, fields in iterate_schema(input_, p['input_schema']):
            # copy referenced files to upload dir
            if field_schema['type'] == "basic:file:":
                old_path = os.path.join(self.current_path, 'inputs', fields[field_schema['name']])
                shutil.copy2(old_path, settings.FILE_UPLOAD_TEMP_DIR)
                file_name = os.path.basename(fields[field_schema['name']])
                fields[field_schema['name']] = {
                    'file': file_name,
                    'file_temp': file_name,
                    'is_remote': False,
                }

            # convert primary keys to strings
            if field_schema['type'].startswith('data:'):
                fields[field_schema['name']] = str(fields[field_schema['name']])
            if field_schema['type'].startswith('list:data:'):
                fields[field_schema['name']] = [str(obj) for obj in fields[field_schema['name']]]

            # fill field with default values if empty
            if 'default' in field_schema and field_schema['name'] not in fields:
                fields[field_schema['name']] = field_schema['default']

        d = Data(
            input=input_,
            author_id=self.user.pk,
            processor_name=processor_name,
            case_ids=[self.case.pk],
        )
        d.save()
        manager(run_sync=True)

        return Data.objects.get(pk=d.pk)

    def assertDone(self, obj):
        self.assertEqual(obj.status, 'done')

    def assertFields(self, obj, path, value):
        field = self.get_field(obj['output'], path)
        return self.assertEqual(field, str(value))

    def assertFiles(self, obj, field_path, fn):
        field = self.get_field(obj['output'], field_path)
        output = os.path.join(settings.DATAFS['data_path'], str(obj.pk), field['file'])
        output_hash = hashlib.sha256(open(output).read()).hexdigest()

        wanted = os.path.join(self.current_path, 'outputs', fn)
        if os.path.isfile(wanted):
            wanted_hash = hashlib.sha256(open(wanted).read()).hexdigest()
            return self.assertEqual(wanted_hash, output_hash)

        shutil.copyfile(output, wanted)
        self.created_files.append(fn)

    def assertJSON(self, storage_id, field_path, fn):
        storage = Storage.objects.get(pk=storage_id)

        field = self.get_field(storage['json'], field_path)
        field_hash = hashlib.sha256(field).hexdigest()

        wanted = os.path.join(self.current_path, 'outputs', fn)
        wanted_hash = hashlib.sha256(open(wanted).read()).hexdigest()

        return self.assertEqual(wanted_hash, field_hash)
<|endoftext|>


<commit_before>from datetime import datetime
from email.utils import parsedate_tz, mktime_tz
import mimetypes
from time import gmtime, time
from os.path import normcase, normpath, join, exists, isdir, getmtime, getsize
from webob.exc import HTTPNotFound, HTTPForbidden, HTTPBadRequest

_BLOCK_SIZE = 4096 * 64 # 256K

mimetypes.init()

class _FileIter(object):
    def __init__(self, file, block_size):
        self.file = file
        self.block_size = block_size

    def __iter__(self):
        return self

    def next(self):
        val = self.file.read(self.block_size)
        if not val:
            raise StopIteration
        return val

    __next__ = next # py3

    def close(self):
        self.file.close()

class FileServeApp(object):
    """
    Serves a static filelike object.
    """
    def __init__(self, path, cache_max_age):
        self.path = path
        self.last_modified = getmtime(path)

        content_type, content_encoding = mimetypes.guess_type(path, strict=False)
        if content_type is None:
            content_type = 'application/octet-stream'

        self.content_type = content_type
        self.content_encoding = content_encoding
        self.content_length = getsize(path)

        if cache_max_age is not None:
            self.cache_expires = cache_max_age

    def generate_etag(self):
        return '"%s-%s"' % (self.last_modified, self.content_length)

    def parse_date(self, value):
        try:
            return mktime_tz(parsedate_tz(value))
        except (TypeError, OverflowError):
            raise HTTPBadRequest(("Received an ill-formed timestamp for %s: %s\r\n") % (self.path, value))

    def make_date(self, d):
        if isinstance(d, datetime):
            d = d.utctimetuple()
        else:
            d = gmtime(d)

        return '%s, %02d%s%s%s%s %02d:%02d:%02d GMT' % (
            ('Mon', 'Tue', 'Wed', 'Thu', 'Fri', 'Sat', 'Sun')[d.tm_wday],
            d.tm_mday, ' ',
            ('Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep',
             'Oct', 'Nov', 'Dec')[d.tm_mon - 1],
            ' ', str(d.tm_year), d.tm_hour, d.tm_min, d.tm_sec)

    def has_been_modified(self, environ, etag, last_modified):
        if environ['REQUEST_METHOD'] not in ('GET', 'HEAD'):
            return False

        unmodified = False

        modified_since = environ.get('HTTP_IF_MODIFIED_SINCE')
        if modified_since:
            modified_since = self.parse_date(modified_since)
            if last_modified and last_modified <= modified_since:
                unmodified = True

        if_none_match = environ.get('HTTP_IF_NONE_MATCH')
        if if_none_match and etag == if_none_match:
            unmodified = True

        return not unmodified

    def __call__(self, environ, start_response):
        try:
            file = open(self.path, 'rb')
        except (IOError, OSError) as e:
            return HTTPForbidden('You are not permitted to view this file (%s)' % e)(start_response, environ)

        headers = []
        timeout = self.cache_expires
        etag = self.generate_etag()
        headers += [('Etag', '%s' % etag),
            ('Cache-Control', 'max-age=%d, public' % timeout)]

        if not self.has_been_modified(environ, etag, self.last_modified):
            file.close()
            start_response('304 Not Modified', headers)
            return []

        headers.extend((
            ('Expires', self.make_date(time() + timeout)),
            ('Content-Type', self.content_type),
            ('Content-Length', str(self.content_length)),
            ('Last-Modified', self.make_date(self.last_modified))
            ))
        start_response('200 OK', headers)
        return _FileIter(file, _BLOCK_SIZE)

INVALID_PATH_PARTS = set(['..', '.']).intersection

class StaticsMiddleware(object):
    def _adapt_path(self, path):
        return normcase(normpath(path))

    def __init__(self, app, root_dir, cache_max_age=3600):
        self.app = app
        self.cache_max_age = cache_max_age
        self.doc_root = self._adapt_path(root_dir)

    def __call__(self, environ, start_response):
        path = environ['PATH_INFO'].split('/')
        if INVALID_PATH_PARTS(path):
            return HTTPNotFound('Out of bounds: %s' % environ['PATH_INFO'])(environ, start_response)

        filepath = self._adapt_path(join(self.doc_root, *path))
        if isdir(filepath) or not exists(filepath):
            return self.app(environ, start_response)

        return FileServeApp(filepath, self.cache_max_age)(environ, start_response)
<commit_msg>Cache path cleanup for statics<commit_after>from datetime import datetime
from email.utils import parsedate_tz, mktime_tz
import mimetypes
from time import gmtime, time
from os.path import normcase, normpath, join, isfile, getmtime, getsize
from webob.exc import HTTPNotFound, HTTPForbidden, HTTPBadRequest
from repoze.lru import LRUCache

_BLOCK_SIZE = 4096 * 64 # 256K

mimetypes.init()

class _FileIter(object):
    def __init__(self, file, block_size):
        self.file = file
        self.block_size = block_size

    def __iter__(self):
        return self

    def next(self):
        val = self.file.read(self.block_size)
        if not val:
            raise StopIteration
        return val

    __next__ = next # py3

    def close(self):
        self.file.close()

class FileServeApp(object):
    """
    Serves a static filelike object.
    """
    def __init__(self, path, cache_max_age):
        self.path = path
        self.last_modified = getmtime(path)

        content_type, content_encoding = mimetypes.guess_type(path, strict=False)
        if content_type is None:
            content_type = 'application/octet-stream'

        self.content_type = content_type
        self.content_encoding = content_encoding
        self.content_length = getsize(path)

        if cache_max_age is not None:
            self.cache_expires = cache_max_age

    def generate_etag(self):
        return '"%s-%s"' % (self.last_modified, self.content_length)

    def parse_date(self, value):
        try:
            return mktime_tz(parsedate_tz(value))
        except (TypeError, OverflowError):
            raise HTTPBadRequest(("Received an ill-formed timestamp for %s: %s\r\n") % (self.path, value))

    def make_date(self, d):
        if isinstance(d, datetime):
            d = d.utctimetuple()
        else:
            d = gmtime(d)

        return '%s, %02d%s%s%s%s %02d:%02d:%02d GMT' % (
            ('Mon', 'Tue', 'Wed', 'Thu', 'Fri', 'Sat', 'Sun')[d.tm_wday],
            d.tm_mday, ' ',
            ('Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep',
             'Oct', 'Nov', 'Dec')[d.tm_mon - 1],
            ' ', str(d.tm_year), d.tm_hour, d.tm_min, d.tm_sec)

    def has_been_modified(self, environ, etag, last_modified):
        if environ['REQUEST_METHOD'] not in ('GET', 'HEAD'):
            return False

        unmodified = False

        modified_since = environ.get('HTTP_IF_MODIFIED_SINCE')
        if modified_since:
            modified_since = self.parse_date(modified_since)
            if last_modified and last_modified <= modified_since:
                unmodified = True

        if_none_match = environ.get('HTTP_IF_NONE_MATCH')
        if if_none_match and etag == if_none_match:
            unmodified = True

        return not unmodified

    def __call__(self, environ, start_response):
        try:
            file = open(self.path, 'rb')
        except (IOError, OSError) as e:
            return HTTPForbidden('You are not permitted to view this file (%s)' % e)(start_response, environ)

        headers = []
        timeout = self.cache_expires
        etag = self.generate_etag()
        headers += [('Etag', '%s' % etag),
            ('Cache-Control', 'max-age=%d, public' % timeout)]

        if not self.has_been_modified(environ, etag, self.last_modified):
            file.close()
            start_response('304 Not Modified', headers)
            return []

        headers.extend((
            ('Expires', self.make_date(time() + timeout)),
            ('Content-Type', self.content_type),
            ('Content-Length', str(self.content_length)),
            ('Last-Modified', self.make_date(self.last_modified))
            ))
        start_response('200 OK', headers)
        return _FileIter(file, _BLOCK_SIZE)

INVALID_PATH_PARTS = set(['..', '.']).intersection

class StaticsMiddleware(object):
    def _adapt_path(self, path):
        return normcase(normpath(path))

    def __init__(self, app, root_dir, cache_max_age=3600):
        self.app = app
        self.cache_max_age = cache_max_age
        self.doc_root = self._adapt_path(root_dir)
        self.paths_cache = LRUCache(1024)

    def __call__(self, environ, start_response):
        full_path = environ['PATH_INFO']
        filepath = self.paths_cache.get(full_path)

        if filepath is None:
            path = full_path.split('/')
            if INVALID_PATH_PARTS(path):
                return HTTPNotFound('Out of bounds: %s' % environ['PATH_INFO'])(environ, start_response)
            filepath = self._adapt_path(join(self.doc_root, *path))
            self.paths_cache.put(full_path, filepath)

        if isfile(filepath):
            return FileServeApp(filepath, self.cache_max_age)(environ, start_response)

        return self.app(environ, start_response)

<|endoftext|>


<commit_before>from collections import defaultdict
import socket
import logging

from distributed import Client, LocalCluster
from elfi.inference_task import InferenceTask

logging.getLogger('tornado').setLevel(logging.WARNING)

_globals = defaultdict(lambda: None)
_whitelist = ["client", "inference_task"]

def set_option(**kwargs):
    """Set global environment settings for ELFI

    Parameters
    ----------
    client : dask.distributed client object

    Examples
    --------
    c = Client()
    set(client = c)

    """
    for k in kwargs:
        if k not in _whitelist:
            raise ValueError("Unrecognized ELFI environment setting {}".format(k))
        _globals[k] = kwargs[k]

def get_option(key):
    if key not in _whitelist:
        raise ValueError("Unrecognized ELFI environment setting {}".format(key))
    return _globals[key]

def clear_option(key):
    if key not in _whitelist:
        raise ValueError("Unrecognized ELFI environment setting {}".format(key))
    if key in _globals:
        del _globals[key]

def client(n_workers=None, threads_per_worker=None):
    """Gets the current framework client, or constructs a local one using the
    parameters if none is found.

    Parameters
    ----------
    n_workers : int
    threads_per_worker : int

    Returns
    -------
    `distributed.Client`
    """

    c = get_option("client")
    if c is None or c.status != 'running':
        cluster_kwargs = {"n_workers": n_workers,
                          "threads_per_worker": threads_per_worker,
                          # Do not start the diagnostics server (bokeh) by default
                          "diagnostics_port": None
                          }
        try:
            cluster = LocalCluster(**cluster_kwargs)
        except (OSError, socket.error):
            # Try with a random port
            cluster_kwargs["scheduler_port"] = 0
            cluster = LocalCluster(**cluster_kwargs)
        c = Client(cluster, set_as_default=True)
        set_option(client = c)
    return c

def inference_task(default=None):
    """Returns the current `InferenceTask`. If default class is provided
    creates a new instance with a given name if none is found.

    Parameters
    ----------
    default : InferenceTask (optional)
       sets default as the current inference task

    Returns
    -------
    `InferenceTask`

    """
    if default is not None:
        if not isinstance(default, InferenceTask):
            raise ValueError("Parameter default must be of type InferenceTask")
        set_option(inference_task=default)

    itask = get_option("inference_task")
    if itask is None:
        # Create a new default inference task
        itask = InferenceTask()
        set_option(inference_task=itask)
    return itask

def new_inference_task(*args, **kwargs):
    """Sets a new inference task for the environment.

    Useful when you want to start anew without worrying about previous node names or
    results.

    Examples
    --------
    p = Prior('p', 'uniform')
    # `p` will now be left to the earlier inference task
    env.new_inference_task()
    # Now we can create a new node with the same name, and it will go to our new
    # inference task
    p = Prior('p', 'uniform')

    Parameters
    ----------
    The parameters will be passed to `InferenceTask`

    Returns
    -------
    `InferenceTask`
    """

    itask = InferenceTask(*args, **kwargs)
    return inference_task(itask)
<commit_msg>Update notebook<commit_after>from collections import defaultdict
import socket
import logging

from distributed import Client, LocalCluster
from elfi.inference_task import InferenceTask

logging.getLogger('tornado').setLevel(logging.WARNING)
logging.getLogger('elfi.methods').setLevel(logging.INFO)

_globals = defaultdict(lambda: None)
_whitelist = ["client", "inference_task"]

def set_option(**kwargs):
    """Set global environment settings for ELFI

    Parameters
    ----------
    client : dask.distributed client object

    Examples
    --------
    c = Client()
    set(client = c)

    """
    for k in kwargs:
        if k not in _whitelist:
            raise ValueError("Unrecognized ELFI environment setting {}".format(k))
        _globals[k] = kwargs[k]

def get_option(key):
    if key not in _whitelist:
        raise ValueError("Unrecognized ELFI environment setting {}".format(key))
    return _globals[key]

def clear_option(key):
    if key not in _whitelist:
        raise ValueError("Unrecognized ELFI environment setting {}".format(key))
    if key in _globals:
        del _globals[key]

def client(n_workers=None, threads_per_worker=None):
    """Gets the current framework client, or constructs a local one using the
    parameters if none is found.

    Parameters
    ----------
    n_workers : int
    threads_per_worker : int

    Returns
    -------
    `distributed.Client`
    """

    c = get_option("client")
    if c is None or c.status != 'running':
        cluster_kwargs = {"n_workers": n_workers,
                          "threads_per_worker": threads_per_worker,
                          # Do not start the diagnostics server (bokeh) by default
                          "diagnostics_port": None
                          }
        try:
            cluster = LocalCluster(**cluster_kwargs)
        except (OSError, socket.error):
            # Try with a random port
            cluster_kwargs["scheduler_port"] = 0
            cluster = LocalCluster(**cluster_kwargs)
        c = Client(cluster, set_as_default=True)
        set_option(client = c)
    return c

def inference_task(default=None):
    """Returns the current `InferenceTask`. If default class is provided
    creates a new instance with a given name if none is found.

    Parameters
    ----------
    default : InferenceTask (optional)
       sets default as the current inference task

    Returns
    -------
    `InferenceTask`

    """
    if default is not None:
        if not isinstance(default, InferenceTask):
            raise ValueError("Parameter default must be of type InferenceTask")
        set_option(inference_task=default)

    itask = get_option("inference_task")
    if itask is None:
        # Create a new default inference task
        itask = InferenceTask()
        set_option(inference_task=itask)
    return itask

def new_inference_task(*args, **kwargs):
    """Sets a new inference task for the environment.

    Useful when you want to start anew without worrying about previous node names or
    results.

    Examples
    --------
    p = Prior('p', 'uniform')
    # `p` will now be left to the earlier inference task
    env.new_inference_task()
    # Now we can create a new node with the same name, and it will go to our new
    # inference task
    p = Prior('p', 'uniform')

    Parameters
    ----------
    The parameters will be passed to `InferenceTask`

    Returns
    -------
    `InferenceTask`
    """

    itask = InferenceTask(*args, **kwargs)
    return inference_task(itask)
<|endoftext|>


<commit_before>#       Licensed to the Apache Software Foundation (ASF) under one
#       or more contributor license agreements.  See the NOTICE file
#       distributed with this work for additional information
#       regarding copyright ownership.  The ASF licenses this file
#       to you under the Apache License, Version 2.0 (the
#       "License"); you may not use this file except in compliance
#       with the License.  You may obtain a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#       Unless required by applicable law or agreed to in writing,
#       software distributed under the License is distributed on an
#       "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
#       KIND, either express or implied.  See the License for the
#       specific language governing permissions and limitations
#       under the License.

import difflib
from pylons import app_globals as g #g is a namespace for globally accessable app helpers
from pylons import tmpl_context as context

from ming import schema
from ming.orm import FieldProperty, ForeignIdProperty, Mapper, session
from ming.orm.declarative import MappedClass

from allura.model import VersionedArtifact, Snapshot, Feed, Thread, Post, User, BaseAttachment
from allura.model import Notification, project_orm_session
from allura.model.timeline import ActivityObject
from allura.lib import helpers as h
from allura.lib import utils

config = utils.ConfigProxy(
    common_suffix='forgemail.domain')

class Globals(MappedClass):

    class __mongometa__:
        name = 'wiki-globals'
        session = project_orm_session
        indexes = [ 'app_config_id' ]

    type_s = 'WikiGlobals'
    _id = FieldProperty(schema.ObjectId)
    app_config_id = ForeignIdProperty('AppConfig', if_missing=lambda:context.app.config._id)
    root = FieldProperty(str)

class PageHistory(Snapshot):
    class __mongometa__:
        name='page_history'

    def original(self):
        return Page.query.get(_id=self.artifact_id)

    def authors(self):
        return self.original().authors()

    def shorthand_id(self):
        return '%s#%s' % (self.original().shorthand_id(), self.version)

    def url(self):
        return self.original().url() + '?version=%d' % self.version

    def index(self):
        result = Snapshot.index(self)
        title = '%s (version %d)' % (self.original().title, self.version)
        result.update(
            title=title,
            type_s='WikiPage Snapshot',
            text=self.data.text)
        return result

    @property
    def html_text(self):
        """A markdown processed version of the page text"""
        return g.markdown_wiki.convert(self.data.text)

    @property
    def attachments(self):
        return self.original().attachments

    @property
    def email_address(self):
        return self.original().email_address

class Page(VersionedArtifact, ActivityObject):
    class __mongometa__:
        name='page'
        history_class = PageHistory

    title=FieldProperty(str)
    text=FieldProperty(schema.String, if_missing='')
    viewable_by=FieldProperty([str])
    type_s = 'Wiki'

    @property
    def activity_name(self):
        return 'wiki page %s' % self.title

    def __json__(self):
        return dict(super(Page, self).__json__(),
                    title=self.title,
                    text=self.text,
                    labels=self.labels,
                    attachments=[dict(bytes=attach.length,
                                      url=h.absurl(attach.url())) for attach in self.attachments])

    def commit(self):
        ss = VersionedArtifact.commit(self)
        session(self).flush()
        if self.version > 1:
            v1 = self.get_version(self.version-1)
            v2 = self
            la = [ line + '\n'  for line in v1.text.splitlines() ]
            lb = [ line + '\n'  for line in v2.text.splitlines() ]
            diff = ''.join(difflib.unified_diff(
                    la, lb,
                    'v%d' % v1.version,
                    'v%d' % v2.version))
            description = '
' + diff + '
'
            if v1.title != v2.title:
                subject = '%s renamed page %s to %s' % (
                    context.user.username, v1.title, v2.title)
            else:
                subject = '%s modified page %s' % (
                    context.user.username, self.title)
        else:
            description = self.text
            subject = '%s created page %s' % (
                context.user.username, self.title)
        Feed.post(self, title=None, description=description)
        Notification.post(
            artifact=self, topic='metadata', text=description, subject=subject)
        return ss

    @property
    def email_address(self):
        domain = '.'.join(reversed(self.app.url[1:-1].split('/'))).replace('_', '-')
        return '%s@%s%s' % (self.title.replace('/', '.'), domain, config.common_suffix)

    @property
    def email_subject(self):
        return 'Discussion for %s page' % self.title

    def url(self):
        s = self.app_config.url() + h.urlquote(self.title.encode('utf-8')) + '/'
        if self.deleted:
            s += '?deleted=True'
        return s

    def shorthand_id(self):
        return self.title

    def index(self):
        result = VersionedArtifact.index(self)
        result.update(
            title=self.title,
            version_i=self.version,
            type_s='WikiPage',
            text=self.text)
        return result

    @property
    def attachments(self):
        return WikiAttachment.query.find(dict(artifact_id=self._id, type='attachment'))

    @classmethod
    def upsert(cls, title, version=None):
        """Update page with `title` or insert new page with that name"""
        if version is None:
            #Check for existing page object
            obj = cls.query.get(
                app_config_id=context.app.config._id,
                title=title)
            if obj is None:
                obj = cls(
                    title=title,
                    app_config_id=context.app.config._id,
                    )
                Thread.new(discussion_id=obj.app_config.discussion_id,
                           ref_id=obj.index_id())
            return obj
        else:
            pg = cls.upsert(title)
            HC = cls.__mongometa__.history_class
            ss = HC.query.find({'artifact_id':pg._id, 'version':int(version)}).one()
            return ss

    @classmethod
    def attachment_class(cls):
        return WikiAttachment

    def reply(self, text):
        Feed.post(self, text)
        # Get thread
        thread = Thread.query.get(artifact_id=self._id)
        return Post(
            discussion_id=thread.discussion_id,
            thread_id=thread._id,
            text=text)

    @property
    def html_text(self):
        """A markdown processed version of the page text"""
        return g.markdown_wiki.convert(self.text)

    def authors(self):
        """All the users that have edited this page"""
        def uniq(users):
            t = {}
            for user in users:
                t[user.username] = user.id
            return t.values()
        user_ids = uniq([r.author for r in self.history().all()])
        return User.query.find({'_id':{'$in':user_ids}}).all()

class WikiAttachment(BaseAttachment):
    ArtifactType=Page
    class __mongometa__:
        polymorphic_identity='WikiAttachment'
    attachment_type=FieldProperty(str, if_missing='WikiAttachment')

Mapper.compile_all()
<commit_msg>[#6208] add unique index to wiki app_config_id+title<commit_after>#       Licensed to the Apache Software Foundation (ASF) under one
#       or more contributor license agreements.  See the NOTICE file
#       distributed with this work for additional information
#       regarding copyright ownership.  The ASF licenses this file
#       to you under the Apache License, Version 2.0 (the
#       "License"); you may not use this file except in compliance
#       with the License.  You may obtain a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#       Unless required by applicable law or agreed to in writing,
#       software distributed under the License is distributed on an
#       "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
#       KIND, either express or implied.  See the License for the
#       specific language governing permissions and limitations
#       under the License.

import difflib
from pylons import app_globals as g #g is a namespace for globally accessable app helpers
from pylons import tmpl_context as context

from ming import schema
from ming.orm import FieldProperty, ForeignIdProperty, Mapper, session
from ming.orm.declarative import MappedClass

from allura.model import VersionedArtifact, Snapshot, Feed, Thread, Post, User, BaseAttachment
from allura.model import Notification, project_orm_session
from allura.model.timeline import ActivityObject
from allura.lib import helpers as h
from allura.lib import utils

config = utils.ConfigProxy(
    common_suffix='forgemail.domain')

class Globals(MappedClass):

    class __mongometa__:
        name = 'wiki-globals'
        session = project_orm_session
        indexes = [ 'app_config_id' ]

    type_s = 'WikiGlobals'
    _id = FieldProperty(schema.ObjectId)
    app_config_id = ForeignIdProperty('AppConfig', if_missing=lambda:context.app.config._id)
    root = FieldProperty(str)

class PageHistory(Snapshot):
    class __mongometa__:
        name='page_history'

    def original(self):
        return Page.query.get(_id=self.artifact_id)

    def authors(self):
        return self.original().authors()

    def shorthand_id(self):
        return '%s#%s' % (self.original().shorthand_id(), self.version)

    def url(self):
        return self.original().url() + '?version=%d' % self.version

    def index(self):
        result = Snapshot.index(self)
        title = '%s (version %d)' % (self.original().title, self.version)
        result.update(
            title=title,
            type_s='WikiPage Snapshot',
            text=self.data.text)
        return result

    @property
    def html_text(self):
        """A markdown processed version of the page text"""
        return g.markdown_wiki.convert(self.data.text)

    @property
    def attachments(self):
        return self.original().attachments

    @property
    def email_address(self):
        return self.original().email_address

class Page(VersionedArtifact, ActivityObject):
    class __mongometa__:
        name='page'
        history_class = PageHistory
        unique_indexes = [('app_config_id', 'title')]

    title=FieldProperty(str)
    text=FieldProperty(schema.String, if_missing='')
    viewable_by=FieldProperty([str])
    type_s = 'Wiki'

    @property
    def activity_name(self):
        return 'wiki page %s' % self.title

    def __json__(self):
        return dict(super(Page, self).__json__(),
                    title=self.title,
                    text=self.text,
                    labels=self.labels,
                    attachments=[dict(bytes=attach.length,
                                      url=h.absurl(attach.url())) for attach in self.attachments])

    def commit(self):
        ss = VersionedArtifact.commit(self)
        session(self).flush()
        if self.version > 1:
            v1 = self.get_version(self.version-1)
            v2 = self
            la = [ line + '\n'  for line in v1.text.splitlines() ]
            lb = [ line + '\n'  for line in v2.text.splitlines() ]
            diff = ''.join(difflib.unified_diff(
                    la, lb,
                    'v%d' % v1.version,
                    'v%d' % v2.version))
            description = '
' + diff + '
'
            if v1.title != v2.title:
                subject = '%s renamed page %s to %s' % (
                    context.user.username, v1.title, v2.title)
            else:
                subject = '%s modified page %s' % (
                    context.user.username, self.title)
        else:
            description = self.text
            subject = '%s created page %s' % (
                context.user.username, self.title)
        Feed.post(self, title=None, description=description)
        Notification.post(
            artifact=self, topic='metadata', text=description, subject=subject)
        return ss

    @property
    def email_address(self):
        domain = '.'.join(reversed(self.app.url[1:-1].split('/'))).replace('_', '-')
        return '%s@%s%s' % (self.title.replace('/', '.'), domain, config.common_suffix)

    @property
    def email_subject(self):
        return 'Discussion for %s page' % self.title

    def url(self):
        s = self.app_config.url() + h.urlquote(self.title.encode('utf-8')) + '/'
        if self.deleted:
            s += '?deleted=True'
        return s

    def shorthand_id(self):
        return self.title

    def index(self):
        result = VersionedArtifact.index(self)
        result.update(
            title=self.title,
            version_i=self.version,
            type_s='WikiPage',
            text=self.text)
        return result

    @property
    def attachments(self):
        return WikiAttachment.query.find(dict(artifact_id=self._id, type='attachment'))

    @classmethod
    def upsert(cls, title, version=None):
        """Update page with `title` or insert new page with that name"""
        if version is None:
            #Check for existing page object
            obj = cls.query.get(
                app_config_id=context.app.config._id,
                title=title)
            if obj is None:
                obj = cls(
                    title=title,
                    app_config_id=context.app.config._id,
                    )
                Thread.new(discussion_id=obj.app_config.discussion_id,
                           ref_id=obj.index_id())
            return obj
        else:
            pg = cls.upsert(title)
            HC = cls.__mongometa__.history_class
            ss = HC.query.find({'artifact_id':pg._id, 'version':int(version)}).one()
            return ss

    @classmethod
    def attachment_class(cls):
        return WikiAttachment

    def reply(self, text):
        Feed.post(self, text)
        # Get thread
        thread = Thread.query.get(artifact_id=self._id)
        return Post(
            discussion_id=thread.discussion_id,
            thread_id=thread._id,
            text=text)

    @property
    def html_text(self):
        """A markdown processed version of the page text"""
        return g.markdown_wiki.convert(self.text)

    def authors(self):
        """All the users that have edited this page"""
        def uniq(users):
            t = {}
            for user in users:
                t[user.username] = user.id
            return t.values()
        user_ids = uniq([r.author for r in self.history().all()])
        return User.query.find({'_id':{'$in':user_ids}}).all()

class WikiAttachment(BaseAttachment):
    ArtifactType=Page
    class __mongometa__:
        polymorphic_identity='WikiAttachment'
    attachment_type=FieldProperty(str, if_missing='WikiAttachment')

Mapper.compile_all()
<|endoftext|>


<commit_before>#!/usr/bin/env python

import pymongo
import time

SLEEP_TIME=60 # wait one minute if no data to process

class DWvideoConverter():

  def __init__(self):
    self.mongo_client = pymongo.MongoClient()
    self.db = mongo_client['eumssi_test']
    self.col = db['content_items']
    self.col.create_index("meta.original_format")
    self.col.create_index("processing_state.available_data")

  def get_items(self, limit=1000):
    return self.col.find({'meta.original_format':'DW-video','processing_state.available_data': "metadata"},fields=['meta.original'],limit=limit)

  def put_item(self, item_id, eumssi_meta):
    ''' write eumssi_meta tweet to MongoDB '''
    try:
      print "updated: ", self.col.update({'_id':item_id},{'$set':{'meta.source':eumssi_meta},'addToSet':{'processing_state.available_data': "metadata"}})
      #print item_id
      #print eumssi_meta
    except Exception as e:
      print e

  def convert(self, original):
    e = {} # eumssi
    o=original # shortcut
    try: e['datePublished']   = o['dateText'] #TODO: date field needs to be parsed properly and converted, maybe use publicationDate
    except Exception: pass
    try: e['inLanguage']      = o['language'] #may need language code normalization
    except Exception: pass
    try: e['text']            = o['description']
    except Exception: pass
    #try: e['author']          = o['user']['id_str'] doesn't seem to have author field
    #except Exception: pass
    try: e['httpHigh'] = o['httpHigh']
    except Exception: pass
    try: e['httpMedium'] = o['httpMedium']
    except Exception: pass
    try: e['keywords'] = o['tags']
    except Exception: pass
    try: e['headline'] = o['title']
    except Exception: pass
    return e

def main():
  conv = DWvideoConverter()
  while(True):
    items = conv.get_items()
    print items.count()
    if items.count() == 0: # no items to process
      print "\n\n\nNO MORE ITEMS\n\n\n"
      return #TODO: remove return statement to allow for continuous updates
      time.sleep(SLEEP_TIME)
    for item in items:
      eumssi_meta = conv.convert(item['meta']['original'])
      conv.put_item(item['_id'],eumssi_meta)

if __name__ == '__main__':
  main()
<commit_msg>hopefully fix DW date conversion<commit_after>#!/usr/bin/env python

import pymongo
import time
from datetime import datetime
import json

SLEEP_TIME=60 # wait one minute if no data to process

class DWvideoConverter():

  def __init__(self):
    self.mongo_client = pymongo.MongoClient()
    self.db = mongo_client['eumssi_test']
    self.col = db['content_items']
    self.col.create_index("meta.original_format")
    self.col.create_index("processing_state.available_data")

  def get_items(self, limit=1000):
    return self.col.find({'meta.original_format':'DW-video','processing_state.available_data': "metadata"},fields=['meta.original'],limit=limit)

  def put_item(self, item_id, eumssi_meta):
    ''' write eumssi_meta tweet to MongoDB '''
    try:
      print "updated: ", self.col.update({'_id':item_id},{'$set':{'meta.source':eumssi_meta},'addToSet':{'processing_state.available_data': "metadata"}})
      #print item_id
      #print eumssi_meta
    except Exception as e:
      print e

  def convert(self, original):
    e = {} # eumssi
    o=original # shortcut
    try:
      d = datetime.utcfromtimestamp(json.loads(o['publicationDate'])['$date']/1000) #convert from timestamp in milliseconds
      e['datePublished']   = d
    except Exception: pass
    try: e['inLanguage']      = o['language'] #may need language code normalization
    except Exception: pass
    try: e['text']            = o['description']
    except Exception: pass
    #try: e['author']          = o['user']['id_str'] doesn't seem to have author field
    #except Exception: pass
    try: e['httpHigh'] = o['httpHigh']
    except Exception: pass
    try: e['httpMedium'] = o['httpMedium']
    except Exception: pass
    try: e['keywords'] = o['tags']
    except Exception: pass
    try: e['headline'] = o['title']
    except Exception: pass
    return e

def main():
  conv = DWvideoConverter()
  while(True):
    items = conv.get_items()
    print items.count()
    if items.count() == 0: # no items to process
      print "\n\n\nNO MORE ITEMS\n\n\n"
      return #TODO: remove return statement to allow for continuous updates
      time.sleep(SLEEP_TIME)
    for item in items:
      eumssi_meta = conv.convert(item['meta']['original'])
      conv.put_item(item['_id'],eumssi_meta)

if __name__ == '__main__':
  main()
<|endoftext|>


<commit_before>from datetime import datetime
from urllib.parse import urlparse

from inputoutput.input import InputReader, Writer
from preprocessing.preprocess_util import  re_whitespace, re_html, re_unicode_decimal, MultiProcessor, replace_in_string, \
    remove_unicode, remove_unprintable, replace_whitespaces, replace_html_entities

def get_new_id(generator, seen, prefix=''):
    i = abs(hash(generator))
    id = prefix + str(i)
    while id in seen:
        i += 1
        id = 't' + str(i)
    return id

class ArticlePreprocessor(MultiProcessor):
    def __init__(self, reader: InputReader, writer: Writer, user_writer, seen_ids=set(), seen_urls_titles=set(),
                 seen_authors: dict = None):
        super().__init__(reader, (writer, user_writer))
        if seen_authors is None:
            seen_authors = dict()
        self.seen_urls_titles = seen_urls_titles  # set: url + '' + title
        self.seen_ids = seen_ids
        self.authors = seen_authors  # dict: Name -> author

    ARTICLE_COLUMNS = ('id', 'author_ids', 'description', 'html', 'published_date', 'title', 'link', 'domain',)

    def process(self, raw_data):
        # title
        title = raw_data['Title']

        # id, based on unqiueness of url
        id = get_new_id(raw_data['Link'], self.seen_ids, 'r')
        url_str = raw_data['Link']
        unqique_str = url_str + '' + title
        if unqique_str in self.seen_urls_titles:
            print("Info: skipping article %s:%d" % (self.reader.current_file, self.reader.i))
        self.seen_urls_titles.add(unqique_str)
        self.seen_ids.add(id)

        # published_date
        try:
            published_date_str = raw_data['Publish date'].replace('CET', '')
            published_date_str = published_date_str.replace('CEST', '')
            published_date = datetime.strptime(published_date_str, '%a %b %d %H:%M:%S  %Y')
        except ValueError as e:
            print("Could not parse date %s" % raw_data['Publish date'])
            raise e

        # link
        url = urlparse(url_str)
        link = url.geturl()
        domain = str(url.hostname)
        if domain.startswith('www.'):
            domain = domain[4:len(domain)]
        if domain.endswith('.com'):
            domain = domain[0:len(domain) - 4]

        # author
        author_ids = self.parse_authors(raw_data)

        # description
        raw_description = raw_data['Description']
        description = replace_in_string(raw_description, replace_html_entities)[0]
        # remove unicode
        description = remove_unicode(description)[0]
        # remove unprintable charcters
        description = remove_unprintable(description)[0]
        # html
        is_html = re_html.search(raw_description) is not None
        if is_html:
            finditer = re_unicode_decimal.finditer(description)
            offset = 0
            for m in finditer:
                unicode = (m.group(1) or m.group(2))
                description = str(description[0:m.start() + offset] + description[m.end() + offset:len(description)])
                offset -= (m.end() - m.start())
            assert re_unicode_decimal.search(description) is None
            description = re_html.sub(' ', description)
        # replace whitespaces
        description = replace_whitespaces(description)

        self.processed_obj(0, {
            'id': id,
            'author_ids': author_ids,
            'description': description,
            'html': is_html,
            'published_date': published_date,
            'title': title,
            'link': link,
            'domain': domain
        })

    AUTHOR_COLUMNS = ('id', 'name',)

    def parse_authors(self, raw_data):
        if raw_data['Author'] == '':
            return []
        author_ids = []
        for author_name in raw_data['Author'].split(', '):
            if author_name == '':
                raise ValueError()
            if author_name in self.authors:
                # already found author before
                author = self.authors[author_name]
            else:
                # new author
                author_id = get_new_id(author_name, self.authors.keys(), 'a')
                author = {
                    'id': author_id,
                    'name': author_name
                }
                self.processed_obj(1, author)
                self.authors[author_name] = author
            author_ids.append(author['id'])
        return author_ids
<commit_msg>Removed duplicate article ids<commit_after>from datetime import datetime
from urllib.parse import urlparse

from inputoutput.input import InputReader, Writer
from preprocessing.preprocess_util import  re_whitespace, re_html, re_unicode_decimal, MultiProcessor, replace_in_string, \
    remove_unicode, remove_unprintable, replace_whitespaces, replace_html_entities

def get_new_id(generator: str, seen_ids: set, prefix=''):
    """"Generate a new unique id"""
    while True:
        i = abs(hash(generator))
        id = prefix + str(i)
        if id not in seen_ids:
            break
    # while id in seen:
    #     i += 1
    #     id = prefix + str(i)
    return id

class ArticlePreprocessor(MultiProcessor):
    def __init__(self, reader: InputReader, writer: Writer, user_writer, seen_ids=set(), seen_urls=set(),
                 seen_authors: dict = None):
        super().__init__(reader, (writer, user_writer))
        if seen_authors is None:
            seen_authors = dict()
        self.seen_urls = seen_urls  # set: url + '' + title
        self.seen_ids = seen_ids
        self.authors = seen_authors  # dict: Name -> author

    ARTICLE_COLUMNS = ('id', 'author_ids', 'description', 'html', 'published_date', 'title', 'link', 'domain',)

    def process(self, raw_data):
        # title
        title = raw_data['Title']

        # check if duplicate
        url_str = raw_data['Link']
        unqique_str = url_str
        if unqique_str in self.seen_urls:
            print("Info: skipping article %s:%d" % (self.reader.current_file, self.reader.i))
            return
        self.seen_urls.add(unqique_str)

        # id, based on unqiueness of url
        article_id = get_new_id(url_str, self.seen_ids, 'r')

        # published_date
        try:
            published_date_str = raw_data['Publish date'].replace('CET', '')
            published_date_str = published_date_str.replace('CEST', '')
            published_date = datetime.strptime(published_date_str, '%a %b %d %H:%M:%S  %Y')
        except ValueError as e:
            print("Could not parse date %s" % raw_data['Publish date'])
            raise e

        # link
        url = urlparse(url_str)
        link = url.geturl()
        domain = str(url.hostname)
        if domain.startswith('www.'):
            domain = domain[4:len(domain)]
        if domain.endswith('.com'):
            domain = domain[0:len(domain) - 4]

        # author
        author_ids = self.parse_authors(raw_data)

        # description
        raw_description = raw_data['Description']
        description = replace_in_string(raw_description, replace_html_entities)[0]
        # remove unicode
        description = remove_unicode(description)[0]
        # remove unprintable charcters
        description = remove_unprintable(description)[0]
        # html
        is_html = re_html.search(raw_description) is not None
        if is_html:
            finditer = re_unicode_decimal.finditer(description)
            offset = 0
            for m in finditer:
                unicode = (m.group(1) or m.group(2))
                description = str(description[0:m.start() + offset] + description[m.end() + offset:len(description)])
                offset -= (m.end() - m.start())
            assert re_unicode_decimal.search(description) is None
            description = re_html.sub(' ', description)
        # replace whitespaces
        description = replace_whitespaces(description)

        self.processed_obj(0, {
            'id': article_id,
            'author_ids': author_ids,
            'description': description,
            'html': is_html,
            'published_date': published_date,
            'title': title,
            'link': link,
            'domain': domain
        })

    AUTHOR_COLUMNS = ('id', 'name',)

    def parse_authors(self, raw_data):
        if raw_data['Author'] == '':
            return []
        author_ids = []
        for author_name in raw_data['Author'].split(', '):
            if author_name == '':
                raise ValueError()
            if author_name in self.authors:
                # already found author before
                author = self.authors[author_name]
            else:
                # new author
                author_id = get_new_id(author_name, self.authors.keys(), 'a')
                author = {
                    'id': author_id,
                    'name': author_name
                }
                self.processed_obj(1, author)
                self.authors[author_name] = author
            author_ids.append(author['id'])
        return author_ids
<|endoftext|>


<commit_before><commit_msg>Wait for IOCS to stop<commit_after><|endoftext|>


<commit_before>__version__ = "0.4.1"
<commit_msg>0.4.2<commit_after>"""
django-timepiece is a multi-user application for tracking people's time on projects.
"""

__version_info__ = {
    'major': 0,
    'minor': 4,
    'micro': 2,
    'releaselevel': 'final',
}

def get_version():
    """
    Return the formatted version information
    """
    vers = ["%(major)i.%(minor)i" % __version_info__, ]

    if __version_info__['micro']:
        vers.append(".%(micro)i" % __version_info__)
    if __version_info__['releaselevel'] != 'final':
        vers.append('%(releaselevel)s' % __version_info__)
    return ''.join(vers)

__version__ = get_version()
<|endoftext|>


<commit_before><commit_msg>option to reuse current tab<commit_after><|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""
    tipfy.auth.twitter
    ~~~~~~~~~~~~~~~~~~

    Implementation of Twitter authentication scheme.

    Ported from `tornado.auth`_.

    :copyright: 2009 Facebook.
    :copyright: 2010 tipfy.org.
    :license: Apache License Version 2.0, see LICENSE.txt for more details.
"""
from __future__ import absolute_import
import functools
import logging
import urllib

from google.appengine.api import urlfetch

from tipfy import REQUIRED_VALUE
from tipfy.utils import json_decode, json_encode
from .oauth import OAuthMixin

#: Default configuration values for this module. Keys are:
#:
#: consumer_key
#:     Key provided when you register an application with Twitter.
#:
#: consumer_secret
#:     Secret provided when you register an application with Twitter.
default_config = {
    'consumer_key':    REQUIRED_VALUE,
    'consumer_secret': REQUIRED_VALUE,
}

class TwitterMixin(OAuthMixin):
    """A :class:`tipfy.RequestHandler` mixin that implements Twitter OAuth
    authentication.

    To authenticate with Twitter, register your application with
    Twitter at http://twitter.com/apps. Then copy your Consumer Key and
    Consumer Secret to ``config.py``::

        config['tipfyext.auth.twitter'] = {
            'consumer_key':    'XXXXXXXXXXXXXXX',
            'consumer_secret': 'XXXXXXXXXXXXXXX',
        }

    When your application is set up, you can use the TwitterMixin to
    authenticate the user with Twitter and get access to their stream.
    You must use the mixin on the handler for the URL you registered as your
    application's Callback URL. For example::

        from tipfy import RequestHandler
        from tipfyext.auth.twitter import TwitterMixin
        from tipfyext.session import CookieMixin, SessionMiddleware

        class TwitterHandler(RequestHandler, CookieMixin, TwitterMixin):
            middleware = [SessionMiddleware]

            def get(self):
                if self.request.args.get('oauth_token', None):
                    return self.get_authenticated_user(self._on_auth)

                return self.authorize_redirect()

            def _on_auth(self, user):
                if not user:
                    self.abort(403)

                # Set the user in the session.
                # ...

    The user object returned by get_authenticated_user() includes the
    attributes 'username', 'name', and all of the custom Twitter user
    attributes describe at
    http://apiwiki.twitter.com/Twitter-REST-API-Method%3A-users%C2%A0show
    in addition to 'access_token'. You should save the access token with
    the user; it is required to make requests on behalf of the user later
    with twitter_request().
    """
    _OAUTH_REQUEST_TOKEN_URL = 'http://api.twitter.com/oauth/request_token'
    _OAUTH_ACCESS_TOKEN_URL = 'http://api.twitter.com/oauth/access_token'
    _OAUTH_AUTHORIZE_URL = 'http://api.twitter.com/oauth/authorize'
    _OAUTH_AUTHENTICATE_URL = 'http://api.twitter.com/oauth/authenticate'
    _OAUTH_NO_CALLBACKS = True

    def _twitter_consumer_key(self):
        return self.app.config[__name__]['consumer_key']

    def _twitter_consumer_secret(self):
        return self.app.config[__name__]['consumer_secret']

    def _oauth_consumer_token(self):
        return dict(
            key=self._twitter_consumer_key(),
            secret=self._twitter_consumer_secret())

    def authenticate_redirect(self):
        """Just like authorize_redirect(), but auto-redirects if authorized.

        This is generally the right interface to use if you are using
        Twitter for single-sign on.
        """
        url = self._oauth_request_token_url()
        try:
            response = urlfetch.fetch(url, deadline=10)
        except urlfetch.DownloadError, e:
            logging.exception(e)
            response = None

        return self._on_request_token(self._OAUTH_AUTHENTICATE_URL, None,
            response)

    def twitter_request(self, path, callback=None, access_token=None,
                           post_args=None, **args):
        """Fetches the given API path, e.g., '/statuses/user_timeline/btaylor'

        The path should not include the format (we automatically append
        '.json' and parse the JSON output).

        If the request is a POST, post_args should be provided. Query
        string arguments should be given as keyword arguments.

        All the Twitter methods are documented at
        http://apiwiki.twitter.com/Twitter-API-Documentation.

        Many methods require an OAuth access token which you can obtain
        through authorize_redirect() and get_authenticated_user(). The
        user returned through that process includes an 'access_token'
        attribute that can be used to make authenticated requests via
        this method. Example usage::

            from tipfy import RequestHandler, Response
            from tipfyext.auth.twitter import TwitterMixin
            from tipfyext.session import CookieMixin, SessionMiddleware

            class TwitterHandler(RequestHandler, CookieMixin, TwitterMixin):
                middleware = [SessionMiddleware]

                def get(self):
                    return self.twitter_request(
                        '/statuses/update',
                        post_args={'status': 'Testing Twitter Mixin'},
                        access_token=user['access_token'],
                        callback=self._on_post)

                def _on_post(self, new_entry):
                    if not new_entry:
                        # Call failed; perhaps missing permission?
                        return self.authorize_redirect()

                    return Response('Posted a message!')
        """
        # Add the OAuth resource request signature if we have credentials
        url = 'http://api.twitter.com/1' + path + '.json'
        if access_token:
            all_args = {}
            all_args.update(args)
            all_args.update(post_args or {})
            method = 'POST' if post_args is not None else 'GET'
            oauth = self._oauth_request_parameters(
                url, access_token, all_args, method=method)
            args.update(oauth)

        if args:
            url += '?' + urllib.urlencode(args)

        try:
            if post_args is not None:
                response = urlfetch.fetch(url, method='POST',
                    payload=urllib.urlencode(post_args), deadline=10)
            else:
                response = urlfetch.fetch(url, deadline=10)
        except urlfetch.DownloadError, e:
            logging.exception(e)
            response = None

        if not callback:
            # Don't preprocess the response, just return a bare one.
            return response

        return self._on_twitter_request(callback, response)

    def _on_twitter_request(self, callback, response):
        if not response:
            logging.warning('Could not get Twitter request token.')
            return callback(None)
        elif response.status_code < 200 or response.status_code >= 300:
            logging.warning('Invalid Twitter response (%d): %s',
                response.status_code, response.content)
            return callback(None)

        return callback(json_decode(response.content))

    def _oauth_get_user(self, access_token, callback):
        callback = functools.partial(self._parse_user_response, callback)
        return self.twitter_request(
            '/users/show/' + access_token['screen_name'],
            access_token=access_token, callback=callback)

    def _parse_user_response(self, callback, user):
        if user:
            user['username'] = user['screen_name']

        return callback(user)
<commit_msg>Patch for Twitter /search endpoint. Thanks, till.<commit_after># -*- coding: utf-8 -*-
"""
    tipfy.auth.twitter
    ~~~~~~~~~~~~~~~~~~

    Implementation of Twitter authentication scheme.

    Ported from `tornado.auth`_.

    :copyright: 2009 Facebook.
    :copyright: 2010 tipfy.org.
    :license: Apache License Version 2.0, see LICENSE.txt for more details.
"""
from __future__ import absolute_import
import functools
import logging
import urllib

from google.appengine.api import urlfetch

from tipfy import REQUIRED_VALUE
from tipfy.utils import json_decode, json_encode
from .oauth import OAuthMixin

#: Default configuration values for this module. Keys are:
#:
#: consumer_key
#:     Key provided when you register an application with Twitter.
#:
#: consumer_secret
#:     Secret provided when you register an application with Twitter.
default_config = {
    'consumer_key':    REQUIRED_VALUE,
    'consumer_secret': REQUIRED_VALUE,
}

class TwitterMixin(OAuthMixin):
    """A :class:`tipfy.RequestHandler` mixin that implements Twitter OAuth
    authentication.

    To authenticate with Twitter, register your application with
    Twitter at http://twitter.com/apps. Then copy your Consumer Key and
    Consumer Secret to ``config.py``::

        config['tipfyext.auth.twitter'] = {
            'consumer_key':    'XXXXXXXXXXXXXXX',
            'consumer_secret': 'XXXXXXXXXXXXXXX',
        }

    When your application is set up, you can use the TwitterMixin to
    authenticate the user with Twitter and get access to their stream.
    You must use the mixin on the handler for the URL you registered as your
    application's Callback URL. For example::

        from tipfy import RequestHandler
        from tipfyext.auth.twitter import TwitterMixin
        from tipfyext.session import CookieMixin, SessionMiddleware

        class TwitterHandler(RequestHandler, CookieMixin, TwitterMixin):
            middleware = [SessionMiddleware]

            def get(self):
                if self.request.args.get('oauth_token', None):
                    return self.get_authenticated_user(self._on_auth)

                return self.authorize_redirect()

            def _on_auth(self, user):
                if not user:
                    self.abort(403)

                # Set the user in the session.
                # ...

    The user object returned by get_authenticated_user() includes the
    attributes 'username', 'name', and all of the custom Twitter user
    attributes describe at
    http://apiwiki.twitter.com/Twitter-REST-API-Method%3A-users%C2%A0show
    in addition to 'access_token'. You should save the access token with
    the user; it is required to make requests on behalf of the user later
    with twitter_request().
    """
    _OAUTH_REQUEST_TOKEN_URL = 'http://api.twitter.com/oauth/request_token'
    _OAUTH_ACCESS_TOKEN_URL = 'http://api.twitter.com/oauth/access_token'
    _OAUTH_AUTHORIZE_URL = 'http://api.twitter.com/oauth/authorize'
    _OAUTH_AUTHENTICATE_URL = 'http://api.twitter.com/oauth/authenticate'
    _OAUTH_NO_CALLBACKS = True

    def _twitter_consumer_key(self):
        return self.app.config[__name__]['consumer_key']

    def _twitter_consumer_secret(self):
        return self.app.config[__name__]['consumer_secret']

    def _oauth_consumer_token(self):
        return dict(
            key=self._twitter_consumer_key(),
            secret=self._twitter_consumer_secret())

    def authenticate_redirect(self):
        """Just like authorize_redirect(), but auto-redirects if authorized.

        This is generally the right interface to use if you are using
        Twitter for single-sign on.
        """
        url = self._oauth_request_token_url()
        try:
            response = urlfetch.fetch(url, deadline=10)
        except urlfetch.DownloadError, e:
            logging.exception(e)
            response = None

        return self._on_request_token(self._OAUTH_AUTHENTICATE_URL, None,
            response)

    def twitter_request(self, path, callback=None, access_token=None,
                           post_args=None, **args):
        """Fetches the given API path, e.g., '/statuses/user_timeline/btaylor'

        The path should not include the format (we automatically append
        '.json' and parse the JSON output).

        If the request is a POST, post_args should be provided. Query
        string arguments should be given as keyword arguments.

        All the Twitter methods are documented at
        http://apiwiki.twitter.com/Twitter-API-Documentation.

        Many methods require an OAuth access token which you can obtain
        through authorize_redirect() and get_authenticated_user(). The
        user returned through that process includes an 'access_token'
        attribute that can be used to make authenticated requests via
        this method. Example usage::

            from tipfy import RequestHandler, Response
            from tipfyext.auth.twitter import TwitterMixin
            from tipfyext.session import CookieMixin, SessionMiddleware

            class TwitterHandler(RequestHandler, CookieMixin, TwitterMixin):
                middleware = [SessionMiddleware]

                def get(self):
                    return self.twitter_request(
                        '/statuses/update',
                        post_args={'status': 'Testing Twitter Mixin'},
                        access_token=user['access_token'],
                        callback=self._on_post)

                def _on_post(self, new_entry):
                    if not new_entry:
                        # Call failed; perhaps missing permission?
                        return self.authorize_redirect()

                    return Response('Posted a message!')
        """
        # Add the OAuth resource request signature if we have credentials
        if path == '/search':
            url = 'http://search.twitter.com/search.json'
        else:
            url = 'http://api.twitter.com/1' + path + '.json'

        if access_token:
            all_args = {}
            all_args.update(args)
            all_args.update(post_args or {})
            method = 'POST' if post_args is not None else 'GET'
            oauth = self._oauth_request_parameters(
                url, access_token, all_args, method=method)
            args.update(oauth)

        if args:
            url += '?' + urllib.urlencode(args)

        try:
            if post_args is not None:
                response = urlfetch.fetch(url, method='POST',
                    payload=urllib.urlencode(post_args), deadline=10)
            else:
                response = urlfetch.fetch(url, deadline=10)
        except urlfetch.DownloadError, e:
            logging.exception(e)
            response = None

        if not callback:
            # Don't preprocess the response, just return a bare one.
            return response

        return self._on_twitter_request(callback, response)

    def _on_twitter_request(self, callback, response):
        if not response:
            logging.warning('Could not get Twitter request token.')
            return callback(None)
        elif response.status_code < 200 or response.status_code >= 300:
            logging.warning('Invalid Twitter response (%d): %s',
                response.status_code, response.content)
            return callback(None)

        return callback(json_decode(response.content))

    def _oauth_get_user(self, access_token, callback):
        callback = functools.partial(self._parse_user_response, callback)
        return self.twitter_request(
            '/users/show/' + access_token['screen_name'],
            access_token=access_token, callback=callback)

    def _parse_user_response(self, callback, user):
        if user:
            user['username'] = user['screen_name']

        return callback(user)
<|endoftext|>