<commit_before>#encoding: utf-8

__author__ = 'Daniel Luque <danielluque14@gmail.com>'
__version__ = '1.0.0'

import urllib

try:
    import simplejson
except ImportError:
    try:
        # Python 2.6 and up
        import json as simplejson
    except ImportError:
    	raise PybooruError('Pybooru requires the simplejson library to work')

class Pybooru(object):
	def __init__(self, name=None, siteURL=None):
		self.baseURL = ''

		if siteURL is not None:
			self.siteURL = str(siteURL).lower()
			self.baseURL = siteURL
		elif name is not None:
			self.name = str(name).lower()
			self._site_name(self.name)
		else:
			print PybooruError('siteURL or name invalid')

	def _site_name(self,name):
		self.site_list = {'konachan': 'http://konachan.com',
						  'danbooru': 'http://danbooru.donmai.us',
						  'yandere': 'https://yande.re',
						  'chan-sankaku': 'htpp://chan.sankakucomplex.com',
						  'idol-sankaku': 'http://idol.sankakucomplex.com',
						  '3dbooru': 'http://behoimi.org',
						  'nekobooru': 'http://nekobooru.net'}

		if name in self.site_list.keys():
			self.baseURL = self.site_list[name]
		else:
			print PybooruError('Site name is not valid')

	def _json_load(self, url):
		try:
			self.openURL = urllib.urlopen(url)
			self.request = self.openURL.read()
			self.response = simplejson.loads(self.request)
			return self.response
		except:
			raise PybooruError('Error in _json_load', self.openURL.getcode(), url)

	def _url_build(self, api_url, params=None):
		if params is not None:
			self.url_request = self.baseURL + api_url + params
			self.url_request = self._json_load(self.url_request)
			return self.url_request
		else:
			self.url_request = self.baseURL + api_url
			self.url_request = self._json_load(self.url_request)
			return self.url_request

	def posts(self, tags=None, limit=10, page=1):
		self.posts_url = '/post/index.json?'
		self.params = 'limit=%i&page=%i' % (limit, page)

		if tags is not None:
			self.tags = str(tags)
			self.params += '&tags=%s' % (tags)
			return self._url_build(self.posts_url, self.params)
		else:
			return self._url_build(self.posts_url, self.params)

	def tags(self, name=None, id_=None, limit=100, page=1, order='name', after_id=0):
		self.tags_url = '/tag/index.json?'

		if id_ is not None:
			self.params = 'id=%i' % (id_)
			return self._url_build(self.tags_url, self.params)
		if name is not None:
			self.name = str(name)
			self.params = "name=%s" % (self.name)
			return self._url_build(self.tags_url, self.params)
		else:
			self.params = "limit=%i&page=%i&order=%s&after_id=%i" % (limit, page, order, after_id)
			return self._url_build(self.tags_url, self.params)

	def artists(self, name=None, id_=None, limit=20, order='name', page=1):
		self.artists_url = '/artist/index.json?'
		self.params = 'limit=%i&page%i&order=%s' % (limit, page, order)

		if name is not None:
			self.name = str(name)
			self.params += '&name=%s' % (self.name)
			return self._url_build(self.artists_url, self.params)
		elif id_ is not None:
			self.params = 'id=%i' % (id_)
			return self._url_build(self.artists_url, self.params)
		else:
			return self._url_build(self.artists_url, self.params)

	def comments(self, id_=None):
		self.comments_url = '/comment/show.json?'

		if id_ is not None:
			self.params = 'id=%i' % (id_)
			return self._url_build(self.comments_url, self.params)
		else:
			print PybooruError('id_ attribute is empty')

	def wiki(self, query=None, order='title', limit=20, page=1):
		self.wiki_url = '/wiki/index.json?'
		self.params = 'order=%s&limit=%i&page=%i' % (order, limit, page)

		if query is not None:
			self.query = str(query)
			self.params += '&query=%s' % (self.query)
			return self._url_build(self.wiki_url, self.params)
		else:
			return self._url_build(self.wiki_url, self.params)

	def notes(self, id_=None):
		self.notes_url = '/note/index.json?'

		if id_ is not None:
			self.params = 'post_id=%i' % (id_)
			return self._url_build(self.notes_url, self.params)
		else:
			return self._url_build(self.notes_url)

	def search_notes(self, query=None):
		self.search_notes_url = '/note/search.json?'

		if query is not None:
			self.query = str(query)
			self.params = 'query=%s' % (self.query)
			return self._url_build(self.search_notes_url, self.params)
		else:
			print PybooruError('query attribute is empty')

	def history_notes(self, post_id=None, id_=None, limit=10, page=1):
		self.history_notes_url = '/note/history.json?'

		if post_id is not None:
			self.params = 'post_id=%i' % (post_id)
			return self._url_build(self.history_notes_url, self.params)
		elif id_ is not None:
			self.params = 'id=%i' % (post_id)
			return self._url_build(self.history_notes_url, self.params)
		else:
			self.params = 'limit=%i&page=%i' % (limit, page)
			return self._url_build(self.history_notes_url, self.params)

	def users(self, name=None, id_=None):
		self.users_url = '/user/index.json?'

		if name is not None:
			self.name = str(name)
			self.params = 'name=%s' % (self.name)
			return self._url_build(self.users_url, self.params)
		elif id_ is not None:
			self.params = 'id=%i' % (self.id_)
			return self._url_build(self.users_url, self.params)
		else:
			return self._url_build(self.users_url)

	def forum(self, id_=None):
		self.forum_url = '/forum/index.json?'

		if id_ is not None:
			self.params = 'parent_id%i' % (id_)
			return self._url_build(self.forum_url, self.params)
		else:
			return self._url_build(sel.forum_url)

	def pools(self, query=None, page=1):
		self.pools_url = '/pool/index.json?'

		if query is not None:
			self.query = str(query)
			self.params = 'query=%s' % (self.query)
			return self._url_build(self.pools_url, self.params)
		else:
			self.params = 'page=%i' % (page)
			return self._url_build(self.pools_url, self.params)

	def pools_posts(self, id_=None, page=1):
		self.pools_posts_url = '/pool/show.json?'

		if id_ is not None:
			self.params = 'id=%i&page=%i' % (id_, page)
			return self._url_build(self.pools_posts_url, self.params)
		else:
			print PybooruError('id_ attribute is empty')

	def favorites(self, id_=None):
		self.favorites = '/favorite/list_users.json?'

		if id_ is not None:
			self.params = 'id=%i' % (id_)
			return self._url_build(self.favorites, self.params)
		else:
			print PybooruError('id_ attribute is empty')

class PybooruError(Exception):
	def __init__(self, err_msg, err_code=None, url=None):
		self.err_msg = err_msg

		if err_code is not None and url is not None:
			if err_code == 200:
				self.err_msg = '%s - ERROR CODE:%i - Request was successful - URL: %s' % (self.err_msg, err_code, url)
			elif err_code == 403:
				self.err_msg = '%s - ERROR CODE:%i - Access denied - URL: %s' % (self.err_msg, err_code, url)
			elif err_code == 404:
				self.err_msg = '%s - ERROR CODE:%i - Not found - URL: %s' % (self.err_msg, err_code, url)
			elif err_code == 420:
				self.err_msg = '%s - ERROR CODE:%i - Record could not be saved - URL: %s' % (self.err_msg, err_code, url)
			elif err_code == 421:
				self.err_msg = '%s - ERROR CODE:%i - User is throttled, try again later - URL: %s' % (self.err_msg, err_code, url)
			elif err_code == 422:
				self.err_msg = '%s - ERROR CODE:%i - The resource is locked and cannot be modified - URL: %s' % (self.err_msg, err_code, url)
			elif err_code == 423:
				self.err_msg = '%s - ERROR CODE:%i - Resource already exists - URL: %s' % (self.err_msg, err_code, url)
			elif err_code == 424:
				self.err_msg = '%s - ERROR CODE:%i - The given parameters were invalid - URL: %s' % (self.err_msg, err_code, url)
			elif err_code == 500:
				self.err_msg = '%s - ERROR CODE:%i - Some unknown error occurred on the server - URL: %s' % (self.err_msg, err_code, url)
			elif err_code == 503:
				self.err_msg = '%s - ERROR CODE:%i - Server cannot currently handle the request, try again later - URL: %s' % (self.err_msg, err_code, url)

	def __str__(self):
		return repr(self.err_msg)

if __name__ == '__main__':
	print PybooruError('import this module into your project to use')<commit_msg>Small changes<commit_after>#!/usr/bin/env Python
#encoding: utf-8

"""
	Pybooru is a library for Python for access to API Danbooru based sites.

	Version: 1.0.0
"""

__author__ = 'Daniel Luque <danielluque14@gmail.com>'
__version__ = '1.0.0'

import urllib

try:
    import simplejson
except ImportError:
    try:
        # Python 2.6 and up
        import json as simplejson
    except ImportError:
    	raise PybooruError('Pybooru requires the simplejson library to work')

class Pybooru(object):
	def __init__(self, name=None, siteURL=None):
		self.baseURL = ''

		if siteURL is not None:
			self.siteURL = str(siteURL).lower()
			self.baseURL = siteURL
		elif name is not None:
			self.name = str(name).lower()
			self._site_name(self.name)
		else:
			print PybooruError('siteURL or name invalid')

	def _site_name(self,name):
		"""
			Site name selector
			------------------
			Atributes:
			------------------
			name - site name
		"""

		self.site_list = {'konachan': 'http://konachan.com',
						  'danbooru': 'http://danbooru.donmai.us',
						  'yandere': 'https://yande.re',
						  'chan-sankaku': 'htpp://chan.sankakucomplex.com',
						  'idol-sankaku': 'http://idol.sankakucomplex.com',
						  '3dbooru': 'http://behoimi.org',
						  'nekobooru': 'http://nekobooru.net'}

		if name in self.site_list.keys():
			self.baseURL = self.site_list[name]
		else:
			print PybooruError('Site name is not valid')

	def _json_load(self, url):
		"""
			JSON Loader
			-----------
			Atributes
			-----------
			url - url of JSON load
		"""

		try:
			self.openURL = urllib.urlopen(url)
			self.request = self.openURL.read()
			self.response = simplejson.loads(self.request)
			return self.response
		except:
			raise PybooruError('Error in _json_load', self.openURL.getcode(), url)

	def _url_build(self, api_url, params=None):
		"""
			url Builder for the JSON loader
		"""

		if params is not None:
			self.url_request = self.baseURL + api_url + params
			self.url_request = self._json_load(self.url_request)
			return self.url_request
		else:
			self.url_request = self.baseURL + api_url
			self.url_request = self._json_load(self.url_request)
			return self.url_request

	def posts(self, tags=None, limit=10, page=1):
		self.posts_url = '/post/index.json?'
		self.params = 'limit=%i&page=%i' % (limit, page)

		if tags is not None:
			self.tags = str(tags)
			self.params += '&tags=%s' % (tags)
			return self._url_build(self.posts_url, self.params)
		else:
			return self._url_build(self.posts_url, self.params)

	def tags(self, name=None, id_=None, limit=100, page=1, order='name', after_id=0):
		self.tags_url = '/tag/index.json?'

		if id_ is not None:
			self.params = 'id=%i' % (id_)
			return self._url_build(self.tags_url, self.params)
		if name is not None:
			self.name = str(name)
			self.params = "name=%s" % (self.name)
			return self._url_build(self.tags_url, self.params)
		else:
			self.params = "limit=%i&page=%i&order=%s&after_id=%i" % (limit, page, order, after_id)
			return self._url_build(self.tags_url, self.params)

	def artists(self, name=None, id_=None, limit=20, order='name', page=1):
		self.artists_url = '/artist/index.json?'
		self.params = 'limit=%i&page%i&order=%s' % (limit, page, order)

		if name is not None:
			self.name = str(name)
			self.params += '&name=%s' % (self.name)
			return self._url_build(self.artists_url, self.params)
		elif id_ is not None:
			self.params = 'id=%i' % (id_)
			return self._url_build(self.artists_url, self.params)
		else:
			return self._url_build(self.artists_url, self.params)

	def comments(self, id_=None):
		self.comments_url = '/comment/show.json?'

		if id_ is not None:
			self.params = 'id=%i' % (id_)
			return self._url_build(self.comments_url, self.params)
		else:
			print PybooruError('id_ attribute is empty')

	def wiki(self, query=None, order='title', limit=20, page=1):
		self.wiki_url = '/wiki/index.json?'
		self.params = 'order=%s&limit=%i&page=%i' % (order, limit, page)

		if query is not None:
			self.query = str(query)
			self.params += '&query=%s' % (self.query)
			return self._url_build(self.wiki_url, self.params)
		else:
			return self._url_build(self.wiki_url, self.params)

	def notes(self, id_=None):
		self.notes_url = '/note/index.json?'

		if id_ is not None:
			self.params = 'post_id=%i' % (id_)
			return self._url_build(self.notes_url, self.params)
		else:
			return self._url_build(self.notes_url)

	def search_notes(self, query=None):
		self.search_notes_url = '/note/search.json?'

		if query is not None:
			self.query = str(query)
			self.params = 'query=%s' % (self.query)
			return self._url_build(self.search_notes_url, self.params)
		else:
			print PybooruError('query attribute is empty')

	def history_notes(self, post_id=None, id_=None, limit=10, page=1):
		self.history_notes_url = '/note/history.json?'

		if post_id is not None:
			self.params = 'post_id=%i' % (post_id)
			return self._url_build(self.history_notes_url, self.params)
		elif id_ is not None:
			self.params = 'id=%i' % (post_id)
			return self._url_build(self.history_notes_url, self.params)
		else:
			self.params = 'limit=%i&page=%i' % (limit, page)
			return self._url_build(self.history_notes_url, self.params)

	def users(self, name=None, id_=None):
		self.users_url = '/user/index.json?'

		if name is not None:
			self.name = str(name)
			self.params = 'name=%s' % (self.name)
			return self._url_build(self.users_url, self.params)
		elif id_ is not None:
			self.params = 'id=%i' % (self.id_)
			return self._url_build(self.users_url, self.params)
		else:
			return self._url_build(self.users_url)

	def forum(self, id_=None):
		self.forum_url = '/forum/index.json?'

		if id_ is not None:
			self.params = 'parent_id%i' % (id_)
			return self._url_build(self.forum_url, self.params)
		else:
			return self._url_build(sel.forum_url)

	def pools(self, query=None, page=1):
		self.pools_url = '/pool/index.json?'

		if query is not None:
			self.query = str(query)
			self.params = 'query=%s' % (self.query)
			return self._url_build(self.pools_url, self.params)
		else:
			self.params = 'page=%i' % (page)
			return self._url_build(self.pools_url, self.params)

	def pools_posts(self, id_=None, page=1):
		self.pools_posts_url = '/pool/show.json?'

		if id_ is not None:
			self.params = 'id=%i&page=%i' % (id_, page)
			return self._url_build(self.pools_posts_url, self.params)
		else:
			print PybooruError('id_ attribute is empty')

	def favorites(self, id_=None):
		self.favorites = '/favorite/list_users.json?'

		if id_ is not None:
			self.params = 'id=%i' % (id_)
			return self._url_build(self.favorites, self.params)
		else:
			print PybooruError('id_ attribute is empty')

class PybooruError(Exception):
	def __init__(self, err_msg, err_code=None, url=None):
		self.err_msg = err_msg

		if err_code is not None and url is not None:
			if err_code == 200:
				self.err_msg = '%s - ERROR CODE:%i - Request was successful - URL: %s' % (self.err_msg, err_code, url)
			elif err_code == 403:
				self.err_msg = '%s - ERROR CODE:%i - Access denied - URL: %s' % (self.err_msg, err_code, url)
			elif err_code == 404:
				self.err_msg = '%s - ERROR CODE:%i - Not found - URL: %s' % (self.err_msg, err_code, url)
			elif err_code == 420:
				self.err_msg = '%s - ERROR CODE:%i - Record could not be saved - URL: %s' % (self.err_msg, err_code, url)
			elif err_code == 421:
				self.err_msg = '%s - ERROR CODE:%i - User is throttled, try again later - URL: %s' % (self.err_msg, err_code, url)
			elif err_code == 422:
				self.err_msg = '%s - ERROR CODE:%i - The resource is locked and cannot be modified - URL: %s' % (self.err_msg, err_code, url)
			elif err_code == 423:
				self.err_msg = '%s - ERROR CODE:%i - Resource already exists - URL: %s' % (self.err_msg, err_code, url)
			elif err_code == 424:
				self.err_msg = '%s - ERROR CODE:%i - The given parameters were invalid - URL: %s' % (self.err_msg, err_code, url)
			elif err_code == 500:
				self.err_msg = '%s - ERROR CODE:%i - Some unknown error occurred on the server - URL: %s' % (self.err_msg, err_code, url)
			elif err_code == 503:
				self.err_msg = '%s - ERROR CODE:%i - Server cannot currently handle the request, try again later - URL: %s' % (self.err_msg, err_code, url)

	def __str__(self):
		return repr(self.err_msg)

if __name__ == '__main__':
	print PybooruError('import this module into your project to use')<|endoftext|>


<commit_before>import numpy as np
import scipy.ndimage as ndimage

def map_coordinates_interpolator(image, coords, shape, **kwargs):
    order = kwargs.get('order', 1)
    mode = kwargs.get('mode', 'constant')
    return ndimage.map_coordinates(image, coords, order=order, mode=mode)

def matlab_interpolator(image, coords, shape, **kwargs):
    from pybug.matlab.wrapper import mlab
    params_list = []
    method = kwargs.get('method', 'linear')
    for i in xrange(coords.shape[0]):
        params_list.append(coords[i, ...].reshape(shape))
    # Append interpolation method
    params_list.append(method)
    return mlab.interpn(image, *params_list).flatten()

def warp(image, template_shape, transform,
         interpolator=map_coordinates_interpolator,
         type='constant', order=1):
    # Swap x and y for images
    dims = list(template_shape)
    dims[:2] = dims[1::-1]
    ranges = [np.arange(dim) for dim in dims]

    # Generate regular grid and flatten for every dimension
    grids = np.meshgrid(*ranges)
    grids = [g.reshape(-1, 1) for g in grids]
    grids = np.hstack(grids)

    # Warp grids
    uv = transform.apply(grids).T
    uv = [np.reshape(x, template_shape) for x in uv]

    # Interpolate according to transformed grid
    warped_image = interpolator(image, uv, order, type)
    warped_image = np.nan_to_num(warped_image)

    return warped_image

def warp_image_onto_template_image(image, template_image, transform,
                                   interpolator=map_coordinates_interpolator,
                                   **kwargs):
    """
    Samples an image at all the masked pixel locations in template_image,
    returning a version of template image where all pixels have been sampled.
    :param image: The image that is to be sampled from
    :param template_image: The template image. Defines what pixels will be
        sampled.
    :param transform: The transform that dictates where the pixel locations
        of the template image are mapped to on image. Pixel values will be
        sampled from the mapped location.
    :param interpolator: The interpolator that will be used in resolving
        what value is chosen from the image at the mapped coordinates.
    :param mode:
    :param order:
    :return:
    """
    template_points = template_image.masked_pixel_indices
    points_to_sample = transform.apply(template_points).T
    sampled_pixel_values = interpolator(np.squeeze(image.pixels),
                                        points_to_sample,
                                        template_image.image_shape, **kwargs)
    sampled_pixel_values = np.nan_to_num(sampled_pixel_values)
    return template_image.from_vector(sampled_pixel_values)<commit_msg>Loop over each channel in the image when warping<commit_after>import numpy as np
import scipy.ndimage as ndimage

def map_coordinates_interpolator(image, coords, shape, **kwargs):
    order = kwargs.get('order', 1)
    mode = kwargs.get('mode', 'constant')
    return ndimage.map_coordinates(np.squeeze(image), coords,
                                   order=order, mode=mode)

def matlab_interpolator(image, coords, shape, **kwargs):
    from pybug.matlab.wrapper import mlab
    params_list = []
    method = kwargs.get('method', 'linear')
    for i in xrange(coords.shape[0]):
        params_list.append(coords[i, ...].reshape(shape))
    # Append interpolation method
    params_list.append(method)
    return mlab.interpn(np.squeeze(image), *params_list).flatten()

def warp(image, template_shape, transform,
         interpolator=map_coordinates_interpolator,
         type='constant', order=1):
    # Swap x and y for images
    dims = list(template_shape)
    dims[:2] = dims[1::-1]
    ranges = [np.arange(dim) for dim in dims]

    # Generate regular grid and flatten for every dimension
    grids = np.meshgrid(*ranges)
    grids = [g.reshape(-1, 1) for g in grids]
    grids = np.hstack(grids)

    # Warp grids
    uv = transform.apply(grids).T
    uv = [np.reshape(x, template_shape) for x in uv]

    # Interpolate according to transformed grid
    warped_image = interpolator(image, uv, order, type)
    warped_image = np.nan_to_num(warped_image)

    return warped_image

def warp_image_onto_template_image(image, template_image, transform,
                                   interpolator=map_coordinates_interpolator,
                                   **kwargs):
    """
    Samples an image at all the masked pixel locations in template_image,
    returning a version of template image where all pixels have been sampled.

    :param image: The image that is to be sampled from
    :param template_image: The template image. Defines what pixels will be
        sampled.
    :param transform: The transform that dictates where the pixel locations
        of the template image are mapped to on image. Pixel values will be
        sampled from the mapped location.
    :param interpolator: The interpolator that will be used in resolving
        what value is chosen from the image at the mapped coordinates.
    :param mode:
    :param order:
    :return:
    """
    template_points = template_image.masked_pixel_indices
    points_to_sample = transform.apply(template_points).T

    sampled_pixel_values = np.zeros([template_image.pixels.size,
                                     image.n_channels])
    pixels = image.pixels
    for i in xrange(image.n_channels):
        sampled_pixel_values[..., i] = interpolator(pixels[..., i],
                                                    points_to_sample,
                                                    template_image.image_shape,
                                                    **kwargs)
    sampled_pixel_values = np.nan_to_num(sampled_pixel_values)

    return template_image.from_vector(sampled_pixel_values,
                                      n_channels=image.n_channels)<|endoftext|>


<commit_before>import csv
import re
import mediacloud
import BeautifulSoup #using BeautifulSoup to make dealing with regex simpler
import urllib2
import sys
import lxml.etree
import json

class BaseMeta():

	url = ""
	soup = BeautifulSoup.BeautifulSoup("")

	def __init__(self, url=None, soup=None):
		if url:
			self.url = url
		if soup:
                	self.soup = soup

	def make_soup(self, stories_id=None, url=None, file_name=None):
		self.url = ""
		self.soup = BeautifulSoup.BeautifulSoup("")
		if stories_id:
			api_key = yaml.load(open('config.yaml'))['mediacloud']['api_key']
			mc = mediacloud.api.MediaCloud(api_key)
			story = mc.story(stories_id,raw_1st_download=1)
			self.url = story[u'url']
			self.soup = BeautifulSoup.BeautifulSoup(story[u'raw_first_download_file'])
		elif url:
			self.url = url
			self.soup = BeautifulSoup.BeautifulSoup(urllib2.urlopen(url).read())
		elif file_name:
			self.url = file_name
			self.soup = BeautifulSoup.BeautifulSoup(open(file_name,'r').read())

	byline_tags = []
        byline_javascript = []
	def get_byline(self, byline_tags=None):
		if byline_tags:
			self.byline_tags = byline_tags
		for t in self.byline_tags:
			if self.soup.find(t[0],{t[1]:t[2]}) is not None:
				if t[0]=='meta':
					return self.soup.find(t[0], {t[1]:t[2]})['content']
				else:
					return self.soup.find(t[0], {t[1]:t[2]}).text
                print self.url
		return ''

	url_section_patterns = []
	section_tags = []
	def get_section(self, url_section_patterns=None, section_tags=None): ##currently acts as get_opinio
		if url_section_patterns:
			self.url_section_patterns = url_section_patterns
		if section_tags:
			self.section_tags = section_tags
		for reg in self.url_section_patterns:
			if re.match(reg,self.url):
				return re.match(reg,self.url).group('section')
		for tag in self.section_tags:
			if self.soup.find(tag[0],{tag[1]:tag[2]}) is not None:
				if tag[0]=='meta':
					return self.soup.find(tag[0], {tag[1]:tag[2]})['content']
				else:
					return self.soup.find(tag[0], {tag[1]:tag[2]}).text
                return ''

	opinion_sections = []
	def is_opinion(self, opinion_sections=None):
		if opinion_sections:
			self.opinion_sections = opinion_sections
		return get_section().lower() in opinion_sections

class NYTimesMeta(BaseMeta):

	byline_tags = [['meta','name','author'],['span','class','byline-author'],['meta','name','clmst'],['meta','name','byl'],['h6','class','byline'],['meta','name','CLMST'],['p','class','byline']]

	url_section_patterns = [r'http://www.nytimes.com/[0-9]{4}/[0-9]{2}/[0-9]{2}/(?P<section>[A-Za-z0-9/]+)/']
	section_tags = [['meta','name',re.compile('cg',re.I)]]

	opinion_sections = ['opinion']

class WashingtonPostMeta(BaseMeta):

	byline_tags = [['meta','name','authors'],['span', 'class', 'blog-byline'], ['meta', 'name', 'dc.creator'],['meta','name','DC.creator'],['div','id','byline'],['p','class','posted'],['span','class','pb-byline']]
        def get_byline(self):
                return self.parse_byline(BaseMeta.get_byline(self))
        byline_patterns = [r'\|{1}(\s+)?(?P<byline>[A-Za-z. -]+)']
        def parse_byline(self, byline):
                for reg in self.byline_patterns:
                        if re.match(reg, byline):
                                return re.match(reg, byline).group('byline')
                return byline

	url_section_patterns = [r'http://feeds.washingtonpost.com/c/34656/f/[0-9]+/s/[A-Za-z0-9]+/(sc/[0-9]+/l|l)/0L0Swashingtonpost0N0C(?P<section>[A-Za-z0-9]+?)s0C[A-Za-z0-9]+/story01.htm',r'http://www.washingtonpost.com/wp-dyn/content/article/[0-9]{4}/[0-9]{2}/[0-9]{2}/AR[0-9]+.html?(nav|wprss)=rss_(?P<section>[A-Za-z0-9]+)(/columns$|/industries$|s$|$)']
	section_tags = [['meta', 'name', 'section']]

	opinion_sections = ['opinion','blog']

class WSJMeta(BaseMeta):

	byline_tags = [['meta','name','article.author'],['meta', 'name', 'author'],['h3','class','byline'],['span','id','byline']]

	section_tags = [['meta', 'name', 'article.section']]

class LATimesMeta(BaseMeta):

	byline_tags = [['span', 'class', 'byline'], ['div','class','trb-bylines'], ['address','class','trb_columnistInfo_columnistPortrait'], ['meta', 'name', 'author'],['div','id','mod-article-byline']]
        def get_byline(self):
                byline = BaseMeta.get_byline(self)
#                if byline == '':
 #                       if self.soup.find('p', text=re.compile(r'( ){0,1}--( )?[A-Za-z\. \-]+( ){0,1}')) is not None:
                                #byline = re.match(r'( ){0,1}--( )?(?P<byline>[A-Za-z\. \-]+)( ){0,1}',self.soup.find('p', text=re.compile(r'( ){0,1}--( )?[A-Za-z\. \-]+( ){0,1}'))).group('byline')
                return self.parse_byline(byline)
        byline_patterns = [r'((January|February|March|April|June|July|August|September|October|November|December)( \d\d?, \d\d\d\d\|)(By )?)(?P<byline>[A-Z]{1}[A-Za-z. -]+)($|, Los Angeles Times)']
        def parse_byline(self, byline):
                for reg in self.byline_patterns:
                        if re.match(reg, byline):
                                return re.match(reg, byline).group('byline')
                return byline

	url_section_patterns = [r'http://feeds.latimes.com/~r/(latimes/)*(?P<section>[A-Za-z/]+)/~3']
	section_tags = [['meta', 'name', 'article.section']]

	opinion_sections = ['news/opinion','OpinionLa']

class HuffPoMeta(BaseMeta):

	byline_tags = [['meta','name','author']]
	#def get_byline(self):
	#	return BaseMeta.get_byline(self).split(' and ')

class SalonMeta(BaseMeta):

	byline_tags = [['span','class','byline'],['a','rel','author']]

class DailyBeastMeta(BaseMeta):

	byline_tags = [['meta','name','authors'],['a','class','more-by-author'],['div','class','author1']]

        url_section_patterns = [r'http://(www.thedailybeast.com/|feedproxy.google.com/~r/thedailybeast/)(?P<section>[A-Za-z-]+)/(~3|item)/']

class YaleMeta(BaseMeta):

	byline_tags = [['div','class','entry-authors']]

class PrincetonMeta(BaseMeta):

	byline_tags = [['p','class','byline']]

class ColumbiaMeta(BaseMeta):

	byline_tags = [['div','class','article-authors']]
<commit_msg>Update base_meta.py<commit_after>#TO-DO: Switch to lxml to process js

import csv
import re
import mediacloud
import BeautifulSoup #using BeautifulSoup to make dealing with regex simpler
import urllib2
import sys
import lxml.etree
import json

class BaseMeta():

	url = ""
	soup = BeautifulSoup.BeautifulSoup("")

	def __init__(self, url=None, soup=None):
		if url:
			self.url = url
		if soup:
                	self.soup = soup

	def make_soup(self, stories_id=None, url=None, file_name=None):
		self.url = ""
		self.soup = BeautifulSoup.BeautifulSoup("")
		if stories_id:
			api_key = yaml.load(open('config.yaml'))['mediacloud']['api_key']
			mc = mediacloud.api.MediaCloud(api_key)
			story = mc.story(stories_id,raw_1st_download=1)
                        try:
                                self.url = story[u'url']
                                self.soup = BeautifulSoup.BeautifulSoup(story[u'raw_first_download_file'])
                        except Exception as e:
                                pass
		elif url:
			self.url = url
			self.soup = BeautifulSoup.BeautifulSoup(urllib2.urlopen(url).read())
		elif file_name:
			self.url = file_name
			self.soup = BeautifulSoup.BeautifulSoup(open(file_name,'r').read())

	byline_tags = []
        byline_javascript = []
	def get_byline(self, byline_tags=None):
		if byline_tags:
			self.byline_tags = byline_tags
		for t in self.byline_tags:
			if self.soup.find(t[0],{t[1]:t[2]}) is not None:
				if t[0]=='meta':
					return self.soup.find(t[0], {t[1]:t[2]})['content']
				else:
					return self.soup.find(t[0], {t[1]:t[2]}).text
		return ''

	url_section_patterns = []
	section_tags = []
	def get_section(self, url_section_patterns=None, section_tags=None): ##currently acts as get_opinio
		if url_section_patterns:
			self.url_section_patterns = url_section_patterns
		if section_tags:
			self.section_tags = section_tags
		for reg in self.url_section_patterns:
			if re.match(reg,self.url):
				return re.match(reg,self.url).group('section')
		for tag in self.section_tags:
			if self.soup.find(tag[0],{tag[1]:tag[2]}) is not None:
				if tag[0]=='meta':
					return self.soup.find(tag[0], {tag[1]:tag[2]})['content']
				else:
					return self.soup.find(tag[0], {tag[1]:tag[2]}).text
                return ''

	opinion_sections = []
	def is_opinion(self, opinion_sections=None):
		if opinion_sections:
			self.opinion_sections = opinion_sections
		return get_section().lower() in opinion_sections

class NYTimesMeta(BaseMeta):

	byline_tags = [['meta','name','author'],['span','class','byline-author'],['meta','name','clmst'],['meta','name','byl'],['h6','class','byline'],['meta','name','CLMST'],['p','class','byline']]

	url_section_patterns = [r'http://www.nytimes.com/[0-9]{4}/[0-9]{2}/[0-9]{2}/(?P<section>[A-Za-z0-9/]+)/']
	section_tags = [['meta','name',re.compile('cg',re.I)]]

	opinion_sections = ['opinion']

class WashingtonPostMeta(BaseMeta):

	byline_tags = [['meta','name','authors'],['span', 'class', 'blog-byline'], ['meta', 'name', 'dc.creator'],['meta','name','DC.creator'],['div','id','byline'],['p','class','posted'],['span','class','pb-byline']]
        def get_byline(self):
                return self.parse_byline(BaseMeta.get_byline(self))
        byline_patterns = [r'\|{1}(\s+)?(?P<byline>[A-Za-z. -]+)']
        def parse_byline(self, byline):
                for reg in self.byline_patterns:
                        if re.match(reg, byline):
                                return re.match(reg, byline).group('byline')
                return byline

	url_section_patterns = [r'http://feeds.washingtonpost.com/c/34656/f/[0-9]+/s/[A-Za-z0-9]+/(sc/[0-9]+/l|l)/0L0Swashingtonpost0N0C(?P<section>[A-Za-z0-9]+?)s0C[A-Za-z0-9]+/story01.htm',r'http://www.washingtonpost.com/wp-dyn/content/article/[0-9]{4}/[0-9]{2}/[0-9]{2}/AR[0-9]+.html?(nav|wprss)=rss_(?P<section>[A-Za-z0-9]+)(/columns$|/industries$|s$|$)']
	section_tags = [['meta', 'name', 'section']]

	opinion_sections = ['opinion','blog']

class WSJMeta(BaseMeta):

	byline_tags = [['meta','name','article.author'],['meta', 'name', 'author'],['h3','class','byline'],['span','id','byline']]

	section_tags = [['meta', 'name', 'article.section']]

class LATimesMeta(BaseMeta):

	byline_tags = [['span', 'class', 'byline'], ['div','class','trb-bylines'], ['address','class','trb_columnistInfo_columnistPortrait'], ['meta', 'name', 'author'],['div','id','mod-article-byline']]
        def get_byline(self):
                byline = BaseMeta.get_byline(self)
#                if byline == '':
 #                       if self.soup.find('p', text=re.compile(r'( ){0,1}--( )?[A-Za-z\. \-]+( ){0,1}')) is not None:
                                #byline = re.match(r'( ){0,1}--( )?(?P<byline>[A-Za-z\. \-]+)( ){0,1}',self.soup.find('p', text=re.compile(r'( ){0,1}--( )?[A-Za-z\. \-]+( ){0,1}'))).group('byline')
                return self.parse_byline(byline)
        byline_patterns = [r'((January|February|March|April|June|July|August|September|October|November|December)( \d\d?, \d\d\d\d\|)(By )?)(?P<byline>[A-Z]{1}[A-Za-z. -]+)($|, Los Angeles Times)']
        def parse_byline(self, byline):
                for reg in self.byline_patterns:
                        if re.match(reg, byline):
                                return re.match(reg, byline).group('byline')
                return byline

	url_section_patterns = [r'http://feeds.latimes.com/~r/(latimes/)*(?P<section>[A-Za-z/]+)/~3']
	section_tags = [['meta', 'name', 'article.section']]

	opinion_sections = ['news/opinion','OpinionLa']

class HuffPoMeta(BaseMeta):

	byline_tags = [['meta','name','author']]
	#def get_byline(self):
	#	return BaseMeta.get_byline(self).split(' and ')

class SalonMeta(BaseMeta):

	byline_tags = [['span','class','byline'],['a','rel','author']]

class DailyBeastMeta(BaseMeta):

	byline_tags = [['meta','name','authors'],['a','class','more-by-author'],['div','class','author1']]

        url_section_patterns = [r'http://(www.thedailybeast.com/|feedproxy.google.com/~r/thedailybeast/)(?P<section>[A-Za-z-]+)/(~3|item)/']

class YaleMeta(BaseMeta):

	byline_tags = [['div','class','entry-authors']]

class PrincetonMeta(BaseMeta):

	byline_tags = [['p','class','byline']]

class ColumbiaMeta(BaseMeta):

	byline_tags = [['div','class','article-authors']]
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# Test.
#

import sys
from rpython import conftest
class o:
    view = False
    viewloops = True
conftest.option = o
from rpython.jit.metainterp.test.test_ajit import LLJitMixin

import pytest
from pycket.test.test_larger import parse_file
from pycket.expand import expand, to_ast
from pycket.interpreter import *
from pycket.values import *

class TestLLtype(LLJitMixin):

    def test_countdown(self):
        ast = to_ast(expand("""
(letrec ([countdown (lambda (n) (if (< n 0) 1 (countdown (- n 1))))])
 (countdown 1000))
"""))

        def interp_w():
            val = interpret_one(ast)
            ov = check_one_val(val)
            assert isinstance(ov, W_Fixnum)
            return ov.value

        assert interp_w() == 1

        self.meta_interp(interp_w, [], listcomp=True, listops=True, backendopt=True)

    def test_countdown_loop(self):
        ast = to_ast(expand("""
(let countdown ([n 1000]) (if (< n 0) 1 (countdown (- n 1))))
"""))

        def interp_w():
            val = interpret_one(ast)
            ov = check_one_val(val)
            assert isinstance(ov, W_Fixnum)
            return ov.value

        assert interp_w() == 1

        self.meta_interp(interp_w, [], listcomp=True, listops=True, backendopt=True)

    def test_side_exit(self):
        ast = to_ast(expand("""
(let ()
    (define (countdown n sub2?)
        (if (< n 0) 1
            (if sub2?
                (countdown (- n 2) #f)
                (countdown (- n 1) #t)
                )))
    (countdown 1000 #f)
)
"""))

        def interp_w():
            val = interpret_one(ast)
            ov = check_one_val(val)
            assert isinstance(ov, W_Fixnum)
            return ov.value

        assert interp_w() == 1

        self.meta_interp(interp_w, [], listcomp=True, listops=True, backendopt=True)

    def test_setbang(self):

        ast = to_ast(expand("(let ([n 1000]) (letrec ([countdown (lambda () (if (< n 0) 1 (begin (set! n (- n 1)) (countdown))))]) (countdown)))"))

        def interp_w():
            val = interpret_one(ast)
            ov = check_one_val(val)
            assert isinstance(ov, W_Fixnum)
            return ov.value

        assert interp_w() == 1

        self.meta_interp(interp_w, [], listcomp=True, listops=True, backendopt=True)

    def test_puzzle(self):
        fname = "puzzle.sch"
        ast = parse_file(fname)
        def interp_w():
            val = interpret_one(ast)
            return val

        self.meta_interp(interp_w, [], listcomp=True, listops=True, backendopt=True)

    def test_bubble(self):
        fname = "bubble.sch"
        ast = parse_file(fname)
        def interp_w():
            val = interpret([ast])
            return val

        self.meta_interp(interp_w, [], listcomp=True, listops=True, backendopt=True)

    def test_bubble_unsafe(self):
        fname = "bubble-unsafe.sch"
        ast = parse_file(fname)
        def interp_w():
            val = interpret([ast])
            return val

        self.meta_interp(interp_w, [], listcomp=True, listops=True, backendopt=True)

    def test_bubble_arg(self):
        fname = "bubble-arg.sch"
        ast = parse_file(fname)
        def interp_w():
            val = interpret([ast])
            return val

        self.meta_interp(interp_w, [], listcomp=True, listops=True, backendopt=True)

    def test_pseudoknot(self):
        fname = "nucleic2.sch"
        ast = parse_file(fname)
        def interp_w():
            val = interpret([ast])
            return val

        self.meta_interp(interp_w, [], listcomp=True, listops=True, backendopt=True)

    def test_minik(self):
        fname = "minikanren.sch"
        ast = parse_file(fname)
        def interp_w():
            val = interpret_one(ast)
            return val

        self.meta_interp(interp_w, [], listcomp=True, listops=True, backendopt=True)

    def test_microk(self):
        fname = "microkanren.sch"
        ast = parse_file(fname)
        def interp_w():
            val = interpret_one(ast)
            return val

        self.meta_interp(interp_w, [], listcomp=True, listops=True, backendopt=True)

    def test_earley(self):
        fname = "earley.sch"
        ast = parse_file(fname)
        def interp_w():
            val = interpret_one(ast)
            return val

        self.meta_interp(interp_w, [], listcomp=True, listops=True, backendopt=True)

    def test_triangle(self):
        fname = "triangle.sch"
        ast = parse_file(fname)
        def interp_w():
            val = interpret_one(ast)
            return val

        self.meta_interp(interp_w, [], listcomp=True, listops=True, backendopt=True)

    def test_append(self):
        ast = to_ast(expand("""
(let () (define (append a b)
  (if (null? a)
      b
      (cons (car a) (append (cdr a) b))))
 (append (list 1 2 3 5 6 6 7 7 8 3 4 5 3 5 4 3 5 3 5 3 3 5 4 3) (list 4 5 6)))
"""
))

        def interp_w():
            val = interpret_one(ast)
            ov = check_one_val(val)
            assert isinstance(ov, W_Object)
            return 1

        assert interp_w() == 1

        self.meta_interp(interp_w, [], listcomp=True, listops=True, backendopt=True)

    def test_let_append(self):
        ast = to_ast(expand("""
(let ()
(define (let-append a b)
  (let ([tst (null? a)])
      (if tst
          b
          (cons (car a) (let-append (cdr a) b)))))
 (let-append (list 1 2 3 5 6 6 7 7 8 3 4 5 3 5 4 3 5 3 5 3 3 5 4 3) (list 4 5 6)))
"""
))

    def test_anormal_append(self):
        ast = to_ast(expand("""
(let ()
(define (append-anormal a b)
  (if (null? a)
      b
      (let* ([ca (car a)]
             [cd (cdr a)]
             [ap (append-anormal cd b)])
        (cons ca ap))))
 (append-anormal (list 1 2 3 5 6 6 7 7 8 3 4 5 3 5 4 3 5 3 5 3 3 5 4 3) (list 4 5 6)))
"""
))

        def interp_w():
            val = interpret_one(ast)
            ov = check_one_val(val)
            assert isinstance(ov, W_Object)
            return 1

        assert interp_w() == 1

        self.meta_interp(interp_w, [], listcomp=True, listops=True, backendopt=True)

    def test_ycombinator(self):

        Y = """
      (lambda (f)
        ((lambda (x) (x x))
         (lambda (g)
           (f (lambda (z) ((g g) z))))))
    """
        countdown = """
        (lambda (f)
          (lambda (x)
            (if (< x 0)
                1
                (f (- x 1)))))
     """
        ast = to_ast(expand("((%s %s) 1000)"%(Y,countdown)))

        def interp_w():
            val = interpret_one(ast)
            ov = check_one_val(val)
            assert isinstance(ov, W_Fixnum)
            return val.value

        assert interp_w() == 1

        self.meta_interp(interp_w, [], listcomp=True, listops=True, backendopt=True)

    def test_vector_get_set(self):
        ast = to_ast(expand("""
(letrec ([v (make-vector 1000 0)]
         [fill-square-vector (lambda (n)
                                (if (< n 0)
                                  (vector-ref v 999)
                                  (begin (vector-set! v n (* n n))
                                         (fill-square-vector (- n 1)))))]
         [sum-vector (lambda (n s)
                        (if (< n 0)
                            s
                            (sum-vector (- n 1) (+ s (vector-ref v n)))))])
 (begin (fill-square-vector 999) (sum-vector 999 0)))
"""))

        def interp_w():
            val = interpret_one(ast)
            ov = check_one_val(val)
            assert isinstance(ov, W_Fixnum)
            return val.value

        assert interp_w() == sum(i ** 2 for i in range(1000))

        self.meta_interp(interp_w, [], listcomp=True, listops=True, backendopt=True)

    def _cons_map(self):
        from pycket.expand import expand_string
        from pycket.json import loads

        ast = to_ast(loads(expand_string("""
(foldl + 0 (map (lambda (x) (+ x 1))
        (list 1 2 3 5 6 6 7 7 8 3 4 5 3 5 4 3 5 3 5 3 3 5 4 3
              1 2 3 5 6 6 7 7 8 3 4 5 3 5 4 3 5 3 5 3 3 5 4 3
              1 2 3 5 6 6 7 7 8 3 4 5 3 5 4 3 5 3 5 3 3 5 4 3
              1 2 3 5 6 6 7 7 8 3 4 5 3 5 4 3 5 3 5 3 3 5 4 3
              1 2 3 5 6 6 7 7 8 3 4 5 3 5 4 3 5 3 5 3 3 5 4 3)))
""", wrap=True, stdlib=True)))

        def interp_w():
            val = interpret_one(ast)
            ov = check_one_val(val)
            assert isinstance(ov, W_Fixnum)
            return ov.value
        assert interp_w() == 635

        self.meta_interp(interp_w, [],
                         listcomp=True, listops=True, backendopt=True)

    def test_scons_map(self):
        import pycket.values
        pycket.values._enable_cons_specialization = True
        self._cons_map()

    def test_ucons_map(self):
        import pycket.values
        pycket.values._enable_cons_specialization = False
        self._cons_map()
<commit_msg>change jit test<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# Test.
#

import sys
from rpython import conftest
class o:
    view = False
    viewloops = True
conftest.option = o
from rpython.jit.metainterp.test.test_ajit import LLJitMixin

import pytest
from pycket.test.test_larger import parse_file
from pycket.expand import expand, to_ast
from pycket.interpreter import *
from pycket.values import *

class TestLLtype(LLJitMixin):

    def test_countdown(self):
        ast = to_ast(expand("""
(letrec ([countdown (lambda (n) (if (< n 0) 1 (countdown (- n 1))))])
 (countdown 1000))
"""))

        def interp_w():
            val = interpret_one(ast)
            ov = check_one_val(val)
            assert isinstance(ov, W_Fixnum)
            return ov.value

        assert interp_w() == 1

        self.meta_interp(interp_w, [], listcomp=True, listops=True, backendopt=True)

    def test_countdown_loop(self):
        ast = to_ast(expand("""
(let countdown ([n 1000]) (if (< n 0) 1 (countdown (- n 1))))
"""))

        def interp_w():
            val = interpret_one(ast)
            ov = check_one_val(val)
            assert isinstance(ov, W_Fixnum)
            return ov.value

        assert interp_w() == 1

        self.meta_interp(interp_w, [], listcomp=True, listops=True, backendopt=True)

    def test_side_exit(self):
        ast = to_ast(expand("""
(let ()
    (define (countdown n sub2?)
        (if (< n 0) 1
            (if sub2?
                (countdown (- n 2) #f)
                (countdown (- n 1) #t)
                )))
    (countdown 1000 #f)
)
"""))

        def interp_w():
            val = interpret_one(ast)
            ov = check_one_val(val)
            assert isinstance(ov, W_Fixnum)
            return ov.value

        assert interp_w() == 1

        self.meta_interp(interp_w, [], listcomp=True, listops=True, backendopt=True)

    def test_setbang(self):

        ast = to_ast(expand("(let ([n 1000]) (letrec ([countdown (lambda () (if (< n 0) 1 (begin (set! n (- n 1)) (countdown))))]) (countdown)))"))

        def interp_w():
            val = interpret_one(ast)
            ov = check_one_val(val)
            assert isinstance(ov, W_Fixnum)
            return ov.value

        assert interp_w() == 1

        self.meta_interp(interp_w, [], listcomp=True, listops=True, backendopt=True)

    def test_puzzle(self):
        fname = "puzzle.sch"
        ast = parse_file(fname)
        def interp_w():
            val = interpret_one(ast)
            return val

        self.meta_interp(interp_w, [], listcomp=True, listops=True, backendopt=True)

    def test_bubble(self):
        fname = "bubble.sch"
        ast = parse_file(fname)
        def interp_w():
            val = interpret([ast])
            return val

        self.meta_interp(interp_w, [], listcomp=True, listops=True, backendopt=True)

    def test_bubble_unsafe(self):
        fname = "bubble-unsafe.sch"
        ast = parse_file(fname)
        def interp_w():
            val = interpret([ast])
            return val

        self.meta_interp(interp_w, [], listcomp=True, listops=True, backendopt=True)

    def test_bubble_arg(self):
        fname = "bubble-arg.sch"
        ast = parse_file(fname)
        def interp_w():
            val = interpret([ast])
            return val

        self.meta_interp(interp_w, [], listcomp=True, listops=True, backendopt=True)

    def test_pseudoknot(self):
        fname = "nucleic2.sch"
        ast = parse_file(fname)
        def interp_w():
            val = interpret([ast])
            return val

        self.meta_interp(interp_w, [], listcomp=True, listops=True, backendopt=True)

    def test_minik(self):
        fname = "minikanren.sch"
        ast = parse_file(fname)
        def interp_w():
            val = interpret_one(ast)
            return val

        self.meta_interp(interp_w, [], listcomp=True, listops=True, backendopt=True)

    def test_microk(self):
        fname = "microkanren.sch"
        ast = parse_file(fname)
        def interp_w():
            val = interpret_one(ast)
            return val

        self.meta_interp(interp_w, [], listcomp=True, listops=True, backendopt=True)

    def test_earley(self):
        fname = "earley.sch"
        ast = parse_file(fname)
        def interp_w():
            val = interpret_one(ast)
            return val

        self.meta_interp(interp_w, [], listcomp=True, listops=True, backendopt=True)

    def test_triangle(self):
        fname = "triangle.sch"
        ast = parse_file(fname)
        def interp_w():
            val = interpret_one(ast)
            return val

        self.meta_interp(interp_w, [], listcomp=True, listops=True, backendopt=True)

    def test_append(self):
        ast = to_ast(expand("""
(let () (define (append a b)
  (if (null? a)
      b
      (cons (car a) (append (cdr a) b))))
 (append (list 1 2 3 5 6 6 7 7 8 3 4 5 3 5 4 3 5 3 5 3 3 5 4 3) (list 4 5 6)))
"""
))

        def interp_w():
            val = interpret_one(ast)
            ov = check_one_val(val)
            assert isinstance(ov, W_Object)
            return 1

        assert interp_w() == 1

        self.meta_interp(interp_w, [], listcomp=True, listops=True, backendopt=True)

    def test_let_append(self):
        ast = to_ast(expand("""
(let ()
(define (let-append a b)
  (let ([tst (null? a)])
      (if tst
          b
          (cons (car a) (let-append (cdr a) b)))))
 (let-append (list 1 2 3 5 6 6 7 7 8 3 4 5 3 5 4 3 5 3 5 3 3 5 4 3) (list 4 5 6)))
"""
))

    def test_anormal_append(self):
        ast = to_ast(expand("""
(let ()
(define (append-anormal a b)
  (if (null? a)
      b
      (let* ([ca (car a)]
             [cd (cdr a)]
             [ap (append-anormal cd b)])
        (cons ca ap))))
 (append-anormal (list 1 2 3 5 6 6 7 7 8 3 4 5 3 5 4 3 5 3 5 3 3 5 4 3) (list 4 5 6)))
"""
))

        def interp_w():
            val = interpret_one(ast)
            ov = check_one_val(val)
            assert isinstance(ov, W_Object)
            return 1

        assert interp_w() == 1

        self.meta_interp(interp_w, [], listcomp=True, listops=True, backendopt=True)

    def test_ycombinator(self):

        Y = """
      (lambda (f)
        ((lambda (x) (x x))
         (lambda (g)
           (f (lambda (z) ((g g) z))))))
    """
        countdown = """
        (lambda (f)
          (lambda (x)
            (if (< x 0)
                1
                (f (- x 1)))))
     """
        ast = to_ast(expand("((%s %s) 1000)"%(Y,countdown)))

        def interp_w():
            val = interpret_one(ast)
            ov = check_one_val(val)
            assert isinstance(ov, W_Fixnum)
            return val.value

        assert interp_w() == 1

        self.meta_interp(interp_w, [], listcomp=True, listops=True, backendopt=True)

    def test_vector_get_set(self):
        ast = to_ast(expand("""
(letrec ([v (make-vector 1000 0)]
         [fill-square-vector (lambda (n)
                                (if (< n 0)
                                  (vector-ref v 999)
                                  (begin (vector-set! v n (* n n))
                                         (fill-square-vector (- n 1)))))]
         [sum-vector (lambda (n s)
                        (if (< n 0)
                            s
                            (sum-vector (- n 1) (+ s (vector-ref v n)))))])
 (begin (fill-square-vector 999) (sum-vector 999 0)))
"""))

        def interp_w():
            val = interpret_one(ast)
            ov = check_one_val(val)
            assert isinstance(ov, W_Fixnum)
            return val.value

        assert interp_w() == sum(i ** 2 for i in range(1000))

        self.meta_interp(interp_w, [], listcomp=True, listops=True, backendopt=True)

    def _cons_map(self):
        from pycket.expand import expand_string
        from pycket.json import loads

        ast = to_ast(loads(expand_string("""
(letrec ([range-it (lambda (n a)
                           (if (<= n 0)
                               a
                               (range-it (- n 1) (cons n a))))]
         [range (lambda (n) (range-it n '()))])
  (foldl + 0 (range 1000)))
""", wrap=True, stdlib=True)))

        def interp_w():
            val = interpret_one(ast)
            ov = check_one_val(val)
            assert isinstance(ov, W_Fixnum)
            return ov.value
        assert interp_w() == 500500

        self.meta_interp(interp_w, [],
                         listcomp=True, listops=True, backendopt=True)

    def test_scons_map(self):
        import pycket.values
        pycket.values._enable_cons_specialization = True
        self._cons_map()

    def test_ucons_map(self):
        import pycket.values
        pycket.values._enable_cons_specialization = False
        self._cons_map()
<|endoftext|>


<commit_before># from django.http import HttpResponse
from django.http import HttpResponseRedirect, Http404
from django.template import RequestContext
from django.core.exceptions import *
from django.views.decorators.http import require_http_methods
from django.shortcuts import get_object_or_404
from django.utils.translation import ugettext_lazy as _
from django.utils.datastructures import SortedDict
from django.db import transaction
from django.db.models.query_utils import Q
from django.contrib.auth.decorators import login_required
from django.contrib.auth.views import redirect_to_login
from django.contrib.auth.forms import AdminPasswordChangeForm
from django.contrib.auth.models import User
from django.contrib.contenttypes.models import ContentType

from django.conf import settings
from django.contrib.auth.decorators import login_required
from django.contrib.auth.forms import AdminPasswordChangeForm
from django.contrib.auth.models import User
from django.contrib.auth.views import login as django_login
from django.contrib.auth.views import logout as django_logout
from django.http import HttpResponseRedirect, HttpResponse
from domain.decorators import login_and_domain_required
from webutils import render_to_response

import datahq.shared_code.hqutils as utils

from xformmanager.models import *
# from hq.models import *
# from graphing import dbhelper
# from graphing.models import *
from receiver.models import *
from domain.decorators import login_and_domain_required, domain_admin_required
from program.models import Program
from phone.models import PhoneUserInfo

@login_and_domain_required
def dashboard(request, template_name="hqwebapp/dashboard.html"):
    startdate, enddate = utils.get_dates(request, 7)

    dates = utils.get_date_range(startdate, enddate)
    dates.reverse()

    # we want a structure of:
    # {program: { user: {day: count}}}
    program_data_structure = {}
    program_totals = {}
    time_bound_metadatas = Metadata.objects.filter(timeend__gt=startdate)\
        .filter(timeend__lt=enddate)\
        .filter(formdefmodel__domain=request.user.selected_domain)

    # get a list of programs --> user mappings
    # TODO: this does a lot of sql querying, for ease of readability/writability
    # if it gets slow we should optimize.
    found_meta_ids = []
    for program in Program.objects.filter(domain=request.user.selected_domain):
        program_map = SortedDict()
        program_totals_by_date = {}
        for date in dates:  program_totals_by_date[date] = 0
        program_users = User.objects.filter(program_membership__program=program)
        for program_user in program_users:
            user_date_map = {}
            for date in dates:  user_date_map[date] = 0
            user_phones = PhoneUserInfo.objects.filter(user=program_user)
            for phone in user_phones:
                # this querying will be a lot cleaner when we attach the real
                # phone object to the metadata
                phone_metas = time_bound_metadatas.filter(deviceid=phone.phone.device_id)\
                    .filter(username=phone.username)
                for meta in phone_metas:
                    user_date_map[meta.timeend.date()] += 1
                    program_totals_by_date[meta.timeend.date()] += 1
                    found_meta_ids.append(meta.id)
            program_map[program_user] = user_date_map
        program_map["Grand Total"] = program_totals_by_date

        # set totals for all dates
        program_user_totals = {}
        for user, data in program_map.items():
            program_user_totals[user] = sum([value for value in data.values()])

        # populate data in higher level maps
        program_data_structure[program]=program_map
        program_totals[program]=program_user_totals

    unregistered_metas = time_bound_metadatas.exclude(id__in=found_meta_ids)
    unregistered_map = SortedDict()
    unregistered_totals_by_date = {}
    for date in dates:  unregistered_totals_by_date[date] = 0
    for meta in unregistered_metas:
        if meta.username not in unregistered_map:
            user_date_map = {}
            for date in dates:   user_date_map[date] = 0
            unregistered_map[meta.username] = user_date_map
        unregistered_map[meta.username][meta.timeend.date()] +=1
        unregistered_totals_by_date[meta.timeend.date()] += 1
    unregistered_map["Grand Total"] = unregistered_totals_by_date
    grand_totals = {}
    for user, data in unregistered_map.items():
        grand_totals[user] = sum([value for value in data.values()])

    return render_to_response(request, template_name,
                              {"program_data": program_data_structure,
                               "program_totals": program_totals,
                               "unregistered_data": unregistered_map,
                               "unregistered_totals": grand_totals,
                               "dates": dates,
                               "startdate": startdate,
                               "enddate": enddate})

# original
# @login_and_domain_required
# def dashboard(request, template_name="hqwebapp/dashboard.html"):
#     return render_to_response(request, template_name,
#                               {})

@login_required()
def password_change(req):
    user_to_edit = User.objects.get(id=req.user.id)
    if req.method == 'POST':
        password_form = AdminPasswordChangeForm(user_to_edit, req.POST)
        if password_form.is_valid():
            password_form.save()
            return HttpResponseRedirect('/')
    else:
        password_form = AdminPasswordChangeForm(user_to_edit)
    template_name="password_change.html"
    return render_to_response(req, template_name, {"form" : password_form})

def server_up(req):
    '''View that just returns "success", which can be hooked into server
       monitoring tools like: http://uptime.openacs.org/uptime/'''
    return HttpResponse("success")

def no_permissions(request):
    template_name="hq/no_permission.html"
    return render_to_response(request, template_name, {})

def login(req, template_name="login_and_password/login.html"):
    '''Login to rapidsms'''
    # this view, and the one below, is overridden because
    # we need to set the base template to use somewhere
    # somewhere that the login page can access it.
    req.base_template = settings.BASE_TEMPLATE
    return django_login(req, **{"template_name" : template_name})

def logout(req, template_name="hqwebapp/loggedout.html"):
    '''Logout of rapidsms'''
    req.base_template = settings.BASE_TEMPLATE
    return django_logout(req, **{"template_name" : template_name})<commit_msg>fix bounds of dashboard to include the last day<commit_after>from datetime import timedelta
# from django.http import HttpResponse
from django.http import HttpResponseRedirect, Http404
from django.template import RequestContext
from django.core.exceptions import *
from django.views.decorators.http import require_http_methods
from django.shortcuts import get_object_or_404
from django.utils.translation import ugettext_lazy as _
from django.utils.datastructures import SortedDict
from django.db import transaction
from django.db.models.query_utils import Q
from django.contrib.auth.decorators import login_required
from django.contrib.auth.views import redirect_to_login
from django.contrib.auth.forms import AdminPasswordChangeForm
from django.contrib.auth.models import User
from django.contrib.contenttypes.models import ContentType

from django.conf import settings
from django.contrib.auth.decorators import login_required
from django.contrib.auth.forms import AdminPasswordChangeForm
from django.contrib.auth.models import User
from django.contrib.auth.views import login as django_login
from django.contrib.auth.views import logout as django_logout
from django.http import HttpResponseRedirect, HttpResponse
from domain.decorators import login_and_domain_required
from webutils import render_to_response

import datahq.shared_code.hqutils as utils

from xformmanager.models import *
# from hq.models import *
# from graphing import dbhelper
# from graphing.models import *
from receiver.models import *
from domain.decorators import login_and_domain_required, domain_admin_required
from program.models import Program
from phone.models import PhoneUserInfo

@login_and_domain_required
def dashboard(request, template_name="hqwebapp/dashboard.html"):
    startdate, enddate = utils.get_dates(request, 7)

    dates = utils.get_date_range(startdate, enddate)
    dates.reverse()

    # we want a structure of:
    # {program: { user: {day: count}}}
    program_data_structure = {}
    program_totals = {}
    # add one to the upper bound of the metadata, to include the last day if
    # necessary
    date_upperbound = enddate + timedelta(days=1)
    time_bound_metadatas = Metadata.objects.filter(timeend__gt=startdate)\
        .filter(timeend__lt=date_upperbound)\
        .filter(formdefmodel__domain=request.user.selected_domain)

    # get a list of programs --> user mappings
    # TODO: this does a lot of sql querying, for ease of readability/writability
    # if it gets slow we should optimize.
    found_meta_ids = []
    for program in Program.objects.filter(domain=request.user.selected_domain):
        program_map = SortedDict()
        program_totals_by_date = {}
        for date in dates:  program_totals_by_date[date] = 0
        program_users = User.objects.filter(program_membership__program=program)
        for program_user in program_users:
            user_date_map = {}
            for date in dates:  user_date_map[date] = 0
            user_phones = PhoneUserInfo.objects.filter(user=program_user)
            for phone in user_phones:
                # this querying will be a lot cleaner when we attach the real
                # phone object to the metadata
                phone_metas = time_bound_metadatas.filter(deviceid=phone.phone.device_id)\
                    .filter(username=phone.username)
                for meta in phone_metas:
                    user_date_map[meta.timeend.date()] += 1
                    program_totals_by_date[meta.timeend.date()] += 1
                    found_meta_ids.append(meta.id)
            program_map[program_user] = user_date_map
        program_map["Grand Total"] = program_totals_by_date

        # set totals for all dates
        program_user_totals = {}
        for user, data in program_map.items():
            program_user_totals[user] = sum([value for value in data.values()])

        # populate data in higher level maps
        program_data_structure[program]=program_map
        program_totals[program]=program_user_totals

    unregistered_metas = time_bound_metadatas.exclude(id__in=found_meta_ids)
    unregistered_map = SortedDict()
    unregistered_totals_by_date = {}
    for date in dates:  unregistered_totals_by_date[date] = 0
    for meta in unregistered_metas:
        if meta.username not in unregistered_map:
            user_date_map = {}
            for date in dates:   user_date_map[date] = 0
            unregistered_map[meta.username] = user_date_map
        unregistered_map[meta.username][meta.timeend.date()] +=1
        unregistered_totals_by_date[meta.timeend.date()] += 1
    unregistered_map["Grand Total"] = unregistered_totals_by_date
    grand_totals = {}
    for user, data in unregistered_map.items():
        grand_totals[user] = sum([value for value in data.values()])

    return render_to_response(request, template_name,
                              {"program_data": program_data_structure,
                               "program_totals": program_totals,
                               "unregistered_data": unregistered_map,
                               "unregistered_totals": grand_totals,
                               "dates": dates,
                               "startdate": startdate,
                               "enddate": enddate})

# original
# @login_and_domain_required
# def dashboard(request, template_name="hqwebapp/dashboard.html"):
#     return render_to_response(request, template_name,
#                               {})

@login_required()
def password_change(req):
    user_to_edit = User.objects.get(id=req.user.id)
    if req.method == 'POST':
        password_form = AdminPasswordChangeForm(user_to_edit, req.POST)
        if password_form.is_valid():
            password_form.save()
            return HttpResponseRedirect('/')
    else:
        password_form = AdminPasswordChangeForm(user_to_edit)
    template_name="password_change.html"
    return render_to_response(req, template_name, {"form" : password_form})

def server_up(req):
    '''View that just returns "success", which can be hooked into server
       monitoring tools like: http://uptime.openacs.org/uptime/'''
    return HttpResponse("success")

def no_permissions(request):
    template_name="hq/no_permission.html"
    return render_to_response(request, template_name, {})

def login(req, template_name="login_and_password/login.html"):
    '''Login to rapidsms'''
    # this view, and the one below, is overridden because
    # we need to set the base template to use somewhere
    # somewhere that the login page can access it.
    req.base_template = settings.BASE_TEMPLATE
    return django_login(req, **{"template_name" : template_name})

def logout(req, template_name="hqwebapp/loggedout.html"):
    '''Logout of rapidsms'''
    req.base_template = settings.BASE_TEMPLATE
    return django_logout(req, **{"template_name" : template_name})<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

import sys

if sys.version_info[:2] > (2, 7):
    from urllib.parse import quote as _quote
    from urllib.parse import unquote as _unquote
    from urllib.parse import urlunsplit, urlsplit
else:
    from urllib import quote as _quote
    from urllib import unquote as _unquote
    from urlparse import urlunsplit, urlsplit

import json

json_encoder = json.JSONEncoder()
def _extract_credentials(url):
    """Extract authentication (user name and password) credentials from the
    given URL.

    >>> extract_credentials('http://localhost:5984/_config/')
    ('http://localhost:5984/_config/', None)
    >>> extract_credentials('http://joe:secret@localhost:5984/_config/')
    ('http://localhost:5984/_config/', ('joe', 'secret'))
    >>> extract_credentials('http://joe%40example.com:secret@localhost:5984/_config/')
    ('http://localhost:5984/_config/', ('joe@example.com', 'secret'))
    """
    parts = urlsplit(url)
    netloc = parts[1]
    if '@' in netloc:
        creds, netloc = netloc.split('@')
        credentials = tuple(_unquote(i) for i in creds.split(':'))
        parts = list(parts)
        parts[1] = netloc
    else:
        credentials = None
    return urlunsplit(parts), credentials

def quote(data, safe=b''):
    if isinstance(data, str):
        data = data.encode('utf-8')
    return _quote(data, safe)

def urljoin(base, *path):
    """Assemble a uri based on a base, any number of path segments, and query
    string parameters.

    >>> urljoin('http://example.org', '_all_dbs')
    'http://example.org/_all_dbs'

    A trailing slash on the uri base is handled gracefully:

    >>> urljoin('http://example.org/', '_all_dbs')
    'http://example.org/_all_dbs'

    And multiple positional arguments become path parts:

    >>> urljoin('http://example.org/', 'foo', 'bar')
    'http://example.org/foo/bar'

    All slashes within a path part are escaped:

    >>> urljoin('http://example.org/', 'foo/bar')
    'http://example.org/foo%2Fbar'
    >>> urljoin('http://example.org/', 'foo', '/bar/')
    'http://example.org/foo/%2Fbar%2F'

    >>> urljoin('http://example.org/', None) #doctest:+IGNORE_EXCEPTION_DETAIL
    Traceback (most recent call last):
        ...
    TypeError: argument 2 to map() must support iteration
    """
    if base and base.endswith('/'):
        base = base[:-1]
    retval = [base]

    # build the path
    path = '/'.join([''] + [quote(s) for s in path])
    if path:
        retval.append(path)

    return ''.join(retval)

def as_json(response):
    if "application/json" in response.headers['content-type']:
        return json.loads(response.text)
    raise RuntimeError()

def to_json(doc):
    return json.dumps(doc)

def _path_from_name(name, type):
    """
    Expand a 'design/foo' style name to its full path as a list of
    segments.
    """
    if name.startswith('_'):
        return name.split('/')
    design, name = name.split('/', 1)
    return ['_design', design, type, name]

def _encode_view_options(options):
    """
    Encode any items in the options dict that are sent as a JSON string to a
    view/list function.
    """
    retval = {}
    for name, value in options.items():
        if name in ('key', 'startkey', 'endkey') \
                or not isinstance(value, str):
            value = json_encoder.encode(value)
        retval[name] = value
    return retval
<commit_msg>Minor code style changes on utils docstrings.<commit_after># -*- coding: utf-8 -*-

import sys

if sys.version_info[:2] > (2, 7):
    from urllib.parse import quote as _quote
    from urllib.parse import unquote as _unquote
    from urllib.parse import urlunsplit, urlsplit
else:
    from urllib import quote as _quote
    from urllib import unquote as _unquote
    from urlparse import urlunsplit, urlsplit

import json

json_encoder = json.JSONEncoder()

def _extract_credentials(url):
    """
    Extract authentication (user name and password) credentials from the
    given URL.

    >>> extract_credentials('http://localhost:5984/_config/')
    ('http://localhost:5984/_config/', None)
    >>> extract_credentials('http://joe:secret@localhost:5984/_config/')
    ('http://localhost:5984/_config/', ('joe', 'secret'))
    >>> extract_credentials('http://joe%40example.com:secret@localhost:5984/_config/')
    ('http://localhost:5984/_config/', ('joe@example.com', 'secret'))
    """
    parts = urlsplit(url)
    netloc = parts[1]
    if '@' in netloc:
        creds, netloc = netloc.split('@')
        credentials = tuple(_unquote(i) for i in creds.split(':'))
        parts = list(parts)
        parts[1] = netloc
    else:
        credentials = None
    return urlunsplit(parts), credentials

def quote(data, safe=b''):
    if isinstance(data, str):
        data = data.encode('utf-8')
    return _quote(data, safe)

def urljoin(base, *path):
    """
    Assemble a uri based on a base, any number of path segments, and query
    string parameters.

    >>> urljoin('http://example.org', '_all_dbs')
    'http://example.org/_all_dbs'

    A trailing slash on the uri base is handled gracefully:

    >>> urljoin('http://example.org/', '_all_dbs')
    'http://example.org/_all_dbs'

    And multiple positional arguments become path parts:

    >>> urljoin('http://example.org/', 'foo', 'bar')
    'http://example.org/foo/bar'

    All slashes within a path part are escaped:

    >>> urljoin('http://example.org/', 'foo/bar')
    'http://example.org/foo%2Fbar'
    >>> urljoin('http://example.org/', 'foo', '/bar/')
    'http://example.org/foo/%2Fbar%2F'

    >>> urljoin('http://example.org/', None) #doctest:+IGNORE_EXCEPTION_DETAIL
    Traceback (most recent call last):
        ...
    TypeError: argument 2 to map() must support iteration
    """
    if base and base.endswith('/'):
        base = base[:-1]
    retval = [base]

    # build the path
    path = '/'.join([''] + [quote(s) for s in path])
    if path:
        retval.append(path)

    return ''.join(retval)

def as_json(response):
    if "application/json" in response.headers['content-type']:
        return json.loads(response.text)
    raise RuntimeError()

def to_json(doc):
    return json.dumps(doc)

def _path_from_name(name, type):
    """
    Expand a 'design/foo' style name to its full path as a list of
    segments.
    """
    if name.startswith('_'):
        return name.split('/')
    design, name = name.split('/', 1)
    return ['_design', design, type, name]

def _encode_view_options(options):
    """
    Encode any items in the options dict that are sent as a JSON string to a
    view/list function.
    """
    retval = {}
    for name, value in options.items():
        if name in ('key', 'startkey', 'endkey') \
                or not isinstance(value, str):
            value = json_encoder.encode(value)
        retval[name] = value
    return retval
<|endoftext|>


<commit_before>
import os

BASE_DIR = os.path.dirname(os.path.dirname(
    os.path.dirname(os.path.abspath(__file__))))

INSTALLED_APPS = [
    'django.contrib.admin',
    'django.contrib.auth',
    'django.contrib.contenttypes',
    'django.contrib.sessions',
    'django.contrib.messages',
    'django.contrib.staticfiles',

    'rest_framework',

    'farmers',
]

MIDDLEWARE = [
    'django.middleware.security.SecurityMiddleware',
    'django.contrib.sessions.middleware.SessionMiddleware',
    'django.middleware.common.CommonMiddleware',
    'django.middleware.csrf.CsrfViewMiddleware',
    'django.contrib.auth.middleware.AuthenticationMiddleware',
    'django.contrib.messages.middleware.MessageMiddleware',
    'django.middleware.clickjacking.XFrameOptionsMiddleware',
]

ROOT_URLCONF = 'config.urls'

TEMPLATES = [
    {
        'BACKEND': 'django.template.backends.django.DjangoTemplates',
        'DIRS': [],
        'APP_DIRS': True,
        'OPTIONS': {
            'context_processors': [
                'django.template.context_processors.debug',
                'django.template.context_processors.request',
                'django.contrib.auth.context_processors.auth',
                'django.contrib.messages.context_processors.messages',
            ],
        },
    },
]

WSGI_APPLICATION = 'config.wsgi.application'

AUTH_PASSWORD_VALIDATORS = [
    {
        'NAME': 'django.contrib.auth.password_validation.'
                'UserAttributeSimilarityValidator',
    },
    {
        'NAME': 'django.contrib.auth.password_validation.'
                'MinimumLengthValidator',
    },
    {
        'NAME': 'django.contrib.auth.password_validation.'
                'CommonPasswordValidator',
    },
    {
        'NAME': 'django.contrib.auth.password_validation.'
                'NumericPasswordValidator',
    },
]

LANGUAGE_CODE = 'en-us'

TIME_ZONE = 'UTC'

USE_I18N = True

USE_L10N = True

USE_TZ = True

STATIC_URL = '/static/'

REST_FRAMEWORK = {
    'DEFAULT_FILTER_BACKENDS': (
        'django_filters.rest_framework.DjangoFilterBackend',
    )
}
<commit_msg>Add default DRF settings<commit_after>
import os

BASE_DIR = os.path.dirname(os.path.dirname(
    os.path.dirname(os.path.abspath(__file__))))

INSTALLED_APPS = [
    'django.contrib.admin',
    'django.contrib.auth',
    'django.contrib.contenttypes',
    'django.contrib.sessions',
    'django.contrib.messages',
    'django.contrib.staticfiles',

    'rest_framework',
    'rest_framework.authtoken',

    'farmers',
]

MIDDLEWARE = [
    'django.middleware.security.SecurityMiddleware',
    'django.contrib.sessions.middleware.SessionMiddleware',
    'django.middleware.common.CommonMiddleware',
    'django.middleware.csrf.CsrfViewMiddleware',
    'django.contrib.auth.middleware.AuthenticationMiddleware',
    'django.contrib.messages.middleware.MessageMiddleware',
    'django.middleware.clickjacking.XFrameOptionsMiddleware',
]

ROOT_URLCONF = 'config.urls'

TEMPLATES = [
    {
        'BACKEND': 'django.template.backends.django.DjangoTemplates',
        'DIRS': [],
        'APP_DIRS': True,
        'OPTIONS': {
            'context_processors': [
                'django.template.context_processors.debug',
                'django.template.context_processors.request',
                'django.contrib.auth.context_processors.auth',
                'django.contrib.messages.context_processors.messages',
            ],
        },
    },
]

WSGI_APPLICATION = 'config.wsgi.application'

AUTH_PASSWORD_VALIDATORS = [
    {
        'NAME': 'django.contrib.auth.password_validation.'
                'UserAttributeSimilarityValidator',
    },
    {
        'NAME': 'django.contrib.auth.password_validation.'
                'MinimumLengthValidator',
    },
    {
        'NAME': 'django.contrib.auth.password_validation.'
                'CommonPasswordValidator',
    },
    {
        'NAME': 'django.contrib.auth.password_validation.'
                'NumericPasswordValidator',
    },
]

LANGUAGE_CODE = 'en-us'

TIME_ZONE = 'UTC'

USE_I18N = True

USE_L10N = True

USE_TZ = True

STATIC_URL = '/static/'

REST_FRAMEWORK = {
    'DEFAULT_FILTER_BACKENDS': (
        'django_filters.rest_framework.DjangoFilterBackend',
    ),
    'DEFAULT_PERMISSION_CLASSES': (
        'rest_framework.permissions.IsAuthenticatedOrReadOnly',
    ),
    'DEFAULT_AUTHENTICATION_CLASSES': (
        'rest_framework.authentication.TokenAuthentication',
    )
}
<|endoftext|>


<commit_before>"""NISO SUSHI support"""
from __future__ import absolute_import

from suds.client import Client

def get_sushi_stats_raw(wsdlurl, start_date, end_date, requestor_id=None,
                        requestor_email=None, customer_reference=None,
                        report="JR1", release=4):
    """Get SUSHI stats for a given site in raw XML format.

    :param wsdlurl: URL to SOAP WSDL for this provider
    :param start_date: start date for report (must be first day of a month)
    :param end_date: end date for report (must be last day of a month)
    :param requestor_id: requestor ID as defined by SUSHI protocol
    :param requestor_email: requestor email address, if required by provider
    :param customer_reference: customer reference number as defined by SUSHI
        protocol
    :param report: report type, values defined by SUSHI protocol
    :param release: report release number (should generally be `4`.)

    """
    client = Client(wsdlurl)
    rdef = client.factory.create('ns1:ReportDefinition')

    rdef._Name = report
    rdef._Release = release
    rdef.Filters.UsageDateRange.Begin = start_date
    rdef.Filters.UsageDateRange.End = end_date

    cref = client.factory.create('ns1:CustomerReference')
    cref.ID = customer_reference

    reqr = client.factory.create('ns1:Requestor')
    reqr.ID = requestor_id
    if requestor_email is not None:
        reqr.Email = requestor_email

    report = client.service.GetReport(reqr, cref, rdef)

    return report
<commit_msg>build report from sushi raw data (partial; includes metadata, but not individual publications<commit_after>"""NISO SUSHI support"""
from __future__ import absolute_import

from suds.client import Client
import pycounter.report
import six

def get_sushi_stats_raw(wsdlurl, start_date, end_date, requestor_id=None,
                        requestor_email=None, customer_reference=None,
                        report="JR1", release=4):
    """Get SUSHI stats for a given site in raw XML format.

    :param wsdlurl: URL to SOAP WSDL for this provider
    :param start_date: start date for report (must be first day of a month)
    :param end_date: end date for report (must be last day of a month)
    :param requestor_id: requestor ID as defined by SUSHI protocol
    :param requestor_email: requestor email address, if required by provider
    :param customer_reference: customer reference number as defined by SUSHI
        protocol
    :param report: report type, values defined by SUSHI protocol
    :param release: report release number (should generally be `4`.)

    """
    client = Client(wsdlurl)
    rdef = client.factory.create('ns1:ReportDefinition')

    rdef._Name = report
    rdef._Release = release
    rdef.Filters.UsageDateRange.Begin = start_date
    rdef.Filters.UsageDateRange.End = end_date

    cref = client.factory.create('ns1:CustomerReference')
    cref.ID = customer_reference

    reqr = client.factory.create('ns1:Requestor')
    reqr.ID = requestor_id
    if requestor_email is not None:
        reqr.Email = requestor_email

    report = client.service.GetReport(reqr, cref, rdef)

    return report

def get_report(*args, **kwargs):
    raw_report = get_sushi_stats_raw(*args, **kwargs)
    return _raw_to_full(raw_report)

def _raw_to_full(raw_report):
    """Convert a raw report to a pycounter.report.CounterReport object"""
    startdate = raw_report.ReportDefinition.Filters.UsageDateRange.Begin
    enddate = raw_report.ReportDefinition.Filters.UsageDateRange.End
    report_data = {}
    report_data['period'] = (startdate, enddate)

    report_data['report_version'] = raw_report.ReportDefinition._Release
    report_data['report_type'] = raw_report.ReportDefinition._Name

    report_data['customer'] = raw_report.Report.Report[0].Customer[0].Name
    report_data['institutional_identifier'] = raw_report.Report.Report[0].Customer[0].ID

    report_data['date_run'] = raw_report.Report.Report[0]._Created.date()

    report = pycounter.report.CounterReport()

    for k, v in six.iteritems(report_data):
        setattr(report, k, v)

    report.metric = pycounter.report.METRICS.get(report_data['report_type'])

    return report

<|endoftext|>