<commit_before>#
# In the first Lesson of the class we are attempting to
# build a Poker program.
#

def poker(hands):
    "Return the best hand: poker([hand,...]) => hand"
    return max(hands, key=hand_rank)

def hand_rank(hand):
    ranks = card_ranks(hand)
    if straight(ranks) and flush(hand):            # straight flush
        return (8, max(ranks))
    elif kind(4, ranks):                           # 4 of a kind
        return (7, kind(4, ranks), kind(1, ranks))
    elif kind(3, ranks) and kind(2, ranks):        # full house
        return (6, kind(3, ranks), kind(2, ranks))
    elif flush(hand):                              # flush
        return (5, ranks)
    elif straight(ranks):                          # straight
        return (4, max(hand))
    elif kind(3, ranks):                           # 3 of a kind
        return (3, kind(3, ranks), ranks)
    elif two_pair(ranks):                          # 2 pair
        return (2, two_pair(ranks))
    elif kind(2, ranks):                           # kind
        return (1, ranks)
    else:                                          # high card
        return (0, ranks)

def test():
    "Test cases for the functions in poker program"
    sf = "6C 7C 8C 9C TC".split() # Straight Flush
    fk = "9D 9H 9S 9C 7D".split() # Four of a Kind
    fh = "TD TC TH 7C 7D".split() # Full House
    assert card_ranks(sf) == [10, 9, 8, 7, 6]
    assert card_ranks(fk) == [9, 9, 9, 9, 7]
    assert card_ranks(fh) == [10, 10, 10, 7, 7]
    assert poker([sf, fk, fh]) == sf
    assert poker([fk, fh]) == fk
    assert poker([fh, fh]) == fh
    assert poker([sf]) == sf
    assert poker([sf] + 99*[fh]) == sf
    assert hand_rank(sf) == (8, 10)
    assert hand_rank(fk) == (7, 9, 7)
    assert hand_rank(fh) == (6, 10, 7)
    return 'tests pass'
<commit_msg>Add card_ranks method.<commit_after>#
# In the first Lesson of the class we are attempting to
# build a Poker program.
#

def poker(hands):
    "Return the best hand: poker([hand,...]) => hand"
    return max(hands, key=hand_rank)

def hand_rank(hand):
    ranks = card_ranks(hand)
    if straight(ranks) and flush(hand):            # straight flush
        return (8, max(ranks))
    elif kind(4, ranks):                           # 4 of a kind
        return (7, kind(4, ranks), kind(1, ranks))
    elif kind(3, ranks) and kind(2, ranks):        # full house
        return (6, kind(3, ranks), kind(2, ranks))
    elif flush(hand):                              # flush
        return (5, ranks)
    elif straight(ranks):                          # straight
        return (4, max(hand))
    elif kind(3, ranks):                           # 3 of a kind
        return (3, kind(3, ranks), ranks)
    elif two_pair(ranks):                          # 2 pair
        return (2, two_pair(ranks))
    elif kind(2, ranks):                           # kind
        return (1, ranks)
    else:                                          # high card
        return (0, ranks)

def card_ranks(cards):
    "Return a list of the ranks, sorted with higher first."

    RANK_MAP = dict(zip(["T", "J", "Q", "K", "A"], range(10, 15)))

    def rank_to_int(card):
        r, s = card
        if r in RANK_MAP:
            return RANK_MAP[r]
        else:
            return int(r)

    ranks = map(rank_to_int, cards)
    ranks.sort(reverse=True)
    return ranks

def test():
    "Test cases for the functions in poker program"
    sf = "6C 7C 8C 9C TC".split() # Straight Flush
    fk = "9D 9H 9S 9C 7D".split() # Four of a Kind
    fh = "TD TC TH 7C 7D".split() # Full House
    assert card_ranks(sf) == [10, 9, 8, 7, 6]
    assert card_ranks(fk) == [9, 9, 9, 9, 7]
    assert card_ranks(fh) == [10, 10, 10, 7, 7]
    assert poker([sf, fk, fh]) == sf
    assert poker([fk, fh]) == fk
    assert poker([fh, fh]) == fh
    assert poker([sf]) == sf
    assert poker([sf] + 99*[fh]) == sf
    assert hand_rank(sf) == (8, 10)
    assert hand_rank(fk) == (7, 9, 7)
    assert hand_rank(fh) == (6, 10, 7)
    return 'tests pass'
<|endoftext|>


<commit_before>#!/usr/bin/env python

import StringIO
import sys
import logging
import os
import re
import flask
from flask import Flask, request, jsonify

import geojson
import requests
import json
import mapzen.whosonfirst.geojson
import mapzen.whosonfirst.export
import mapzen.whosonfirst.search
import mapzen.whosonfirst.utils
import mapzen.whosonfirst.placetypes
import mapzen.whosonfirst.pip.utils

app = Flask(__name__)

@app.before_request
def init():
	flask.g.wof_pending_dir = os.environ.get('WOF_PENDING_DIR', '/usr/local/mapzen/whosonfirst-www-boundaryissues/pending/')

@app.route('/encode', methods=['POST'])
def geojson_encode():

	try:
		g = request.form['geojson']
		f = geojson.loads(g)
	except Exception, e:
		error = "failed to load geojson, because %s" % e
		logging.error(error)
		return jsonify(ok=0, error=error)

	# Does the input pass the smell check?
	validation = geojson.is_valid(f)

	if (validation['valid'] == 'no'):
		error = "GeoJSON doesn't smell right: %s" % validation['message']
		logging.error(error)
		return jsonify(ok=0, error=error)

	e = mapzen.whosonfirst.geojson.encoder(precision=None)

	try:
		fh = StringIO.StringIO()
		e.encode_feature(f, fh)
	except Exception, e:
		error = "failed to encode geojson, because %s" % e
		logging.error(error)
		return jsonify(ok=0, error=error)

	fh.seek(0)
	encoded = fh.read()

	return jsonify(ok=1, encoded=encoded)

@app.route('/save', methods=['POST'])
def geojson_save():

	try:
		g = request.form['geojson']
		branch = request.form['branch']
		f = geojson.loads(g)
	except Exception, e:
		err = "failed to load geojson, because %s" % e
		return jsonify(ok=0, error=err)

	p = re.compile('^[a-zA-Z0-9-_]+$')
	if not p.match(branch):
		return jsonify(ok=0, error='Invalid branch name: %s' % branch)

	# Does the input pass the smell check?
	validation = geojson.is_valid(f)

	if (validation['valid'] == 'no'):
		error = "GeoJSON doesn't smell right: %s" % validation['message']
		logging.error(error)
		return jsonify(ok=0, error=error)

	try:
		data_dir = "%s%s/data" % (flask.g.wof_pending_dir, branch)
		if not os.path.exists(data_dir):
			os.makedirs(data_dir, 0775)
		ff = mapzen.whosonfirst.export.flatfile(data_dir, debug=False)
		path = ff.export_feature(f)
	except Exception, e:
		error = "failed to export geojson, because %s" % e
		logging.error(error)
		return jsonify(ok=0, error=error)

	if not path:
		return jsonify(ok=0, error="File export returned an empty path, maybe check file permissions?")

	# Repeat back the file we just wrote
	try:
		gf = open(path, 'r')
		exported = gf.read()
	except Exception, e:
		error = "failed to read exported geojson, because %s" % e
		logging.error(error)
		return jsonify(ok=0, error=error)

	return jsonify(ok=1, geojson=exported)

@app.route('/pip', methods=['GET'])
def geojson_hierarchy():
	data_endpoint = 'https://whosonfirst.mapzen.com/data/'
	lat = float(request.args.get('latitude'))
	lng = float(request.args.get('longitude'))
	wof_id = request.args.get('wof_id')
	placetype = request.args.get('placetype')
	pip_server = mapzen.whosonfirst.pip.server(hostname='pip.mapzen.com', scheme='https', port=443)

	if (mapzen.whosonfirst.placetypes.is_valid_placetype(placetype) == False):
		return jsonify(ok=0, error="What is that placetype?")

	try:
		parents = mapzen.whosonfirst.pip.utils.get_reverse_geocoded(lat, lng, placetype, pip_server=pip_server)
	except Exception, e:
		error = "failed to determine parents, because %s" % e
		logging.error(error)
		return jsonify(ok=0, error=error)

	# Ok, here is a weird situation. Let's say you've got a venue out in the
	# desert. Its usual parents like neighbourhood or microhood don't exist out
	# here. But we still want to build a hierarchy if our first attempt comes
	# up empty. We're just gonna retry, with locality as the new placetype.
	# Eventually this will probably get handled in a Python library, but for now
	# we'll just do it the HULKSMASH way. (20170504/dphiffer)
	if placetype == 'venue' and len(parents) == 0:
		placetype = 'locality'
		parents = mapzen.whosonfirst.pip.utils.get_reverse_geocoded(lat, lng, placetype, pip_server=pip_server)

	try:
		hierarchy = mapzen.whosonfirst.pip.utils.get_hierarchy(parents, wof_id, placetype)
	except Exception, e:
		error = "failed to determine hierarchy, because %s" % e
		logging.error(error)
		return jsonify(ok=0, error=error)

	return jsonify(ok=1, hierarchy=hierarchy, parents=parents)

@app.route('/nearby', methods=['GET'])
def nearby():

	# Please put me in a config file
	api_key = "mapzen-Gvj9yGE"

	lat = float(request.args.get('latitude'))
	lng = float(request.args.get('longitude'))
	name = request.args.get('name')

	results = []
	result_ids = []

	method = 'whosonfirst.places.getByLatLon'
	params = { 'api_key': api_key, 'method': method, 'latitude': lat, 'longitude': lng, 'placetype': 'neighbourhood' }

	rsp = requests.get('https://whosonfirst-api.mapzen.com/', params=params)
	data = json.loads(rsp.content)

	if (data['places'] and len(data['places']) > 0):
		for neighbourhood in data['places']:

			wofid = neighbourhood['wof:id']
			method = 'whosonfirst.places.search'
			params = { 'api_key': api_key, 'method': method, 'neighbourhood_id': wofid, 'names': name, 'extras': 'addr:', 'placetype': 'venue' }

			rsp = requests.get('https://whosonfirst-api.mapzen.com/', params=params)
			data = json.loads(rsp.content)

			if data['places']:
				for pl in data['places']:
					if pl['wof:id'] not in result_ids:
						result_ids.append(pl['wof:id'])
						results.append(pl)

	method = "whosonfirst.places.getNearby"
	params = { 'api_key': api_key, 'method': method, 'latitude': lat, 'longitude': lng, 'placetype': 'venue', 'radius': 30, 'extras': 'addr:full' }

	rsp = requests.get('https://whosonfirst-api.mapzen.com/', params=params)
	print rsp.content
	data = json.loads(rsp.content)

	if (data['places'] and len(data['places']) > 0):
		for pl in data['places']:
			if pl['wof:id'] not in result_ids:
				result_ids.append(pl['wof:id'])
				results.append(pl)

	return jsonify(ok=1, results=results)

if __name__ == "__main__":
	import sys
	import optparse
	import ConfigParser

	opt_parser = optparse.OptionParser()

	opt_parser.add_option('-p', '--port', dest='port', action='store', default=8181, help='')
	opt_parser.add_option('-d', '--dir', dest='dir', action='store', default='/usr/local/mapzen/whosonfirst-www-boundaryissues/pending/', help='wof_pending_dir')
	opt_parser.add_option('-v', '--verbose', dest='verbose', action='store_true', default=False, help='Be chatty (default is false)')

	options, args = opt_parser.parse_args()

	if options.verbose:
		logging.basicConfig(level=logging.DEBUG)
	else:
		logging.basicConfig(level=logging.INFO)

	os.environ['WOF_PENDING_DIR'] = options.dir
	port = int(options.port)

	app.run(port=port)
<commit_msg>:bug: don't clobber placetype var<commit_after>#!/usr/bin/env python

import StringIO
import sys
import logging
import os
import re
import flask
from flask import Flask, request, jsonify

import geojson
import requests
import json
import mapzen.whosonfirst.geojson
import mapzen.whosonfirst.export
import mapzen.whosonfirst.search
import mapzen.whosonfirst.utils
import mapzen.whosonfirst.placetypes
import mapzen.whosonfirst.pip.utils

app = Flask(__name__)

@app.before_request
def init():
	flask.g.wof_pending_dir = os.environ.get('WOF_PENDING_DIR', '/usr/local/mapzen/whosonfirst-www-boundaryissues/pending/')

@app.route('/encode', methods=['POST'])
def geojson_encode():

	try:
		g = request.form['geojson']
		f = geojson.loads(g)
	except Exception, e:
		error = "failed to load geojson, because %s" % e
		logging.error(error)
		return jsonify(ok=0, error=error)

	# Does the input pass the smell check?
	validation = geojson.is_valid(f)

	if (validation['valid'] == 'no'):
		error = "GeoJSON doesn't smell right: %s" % validation['message']
		logging.error(error)
		return jsonify(ok=0, error=error)

	e = mapzen.whosonfirst.geojson.encoder(precision=None)

	try:
		fh = StringIO.StringIO()
		e.encode_feature(f, fh)
	except Exception, e:
		error = "failed to encode geojson, because %s" % e
		logging.error(error)
		return jsonify(ok=0, error=error)

	fh.seek(0)
	encoded = fh.read()

	return jsonify(ok=1, encoded=encoded)

@app.route('/save', methods=['POST'])
def geojson_save():

	try:
		g = request.form['geojson']
		branch = request.form['branch']
		f = geojson.loads(g)
	except Exception, e:
		err = "failed to load geojson, because %s" % e
		return jsonify(ok=0, error=err)

	p = re.compile('^[a-zA-Z0-9-_]+$')
	if not p.match(branch):
		return jsonify(ok=0, error='Invalid branch name: %s' % branch)

	# Does the input pass the smell check?
	validation = geojson.is_valid(f)

	if (validation['valid'] == 'no'):
		error = "GeoJSON doesn't smell right: %s" % validation['message']
		logging.error(error)
		return jsonify(ok=0, error=error)

	try:
		data_dir = "%s%s/data" % (flask.g.wof_pending_dir, branch)
		if not os.path.exists(data_dir):
			os.makedirs(data_dir, 0775)
		ff = mapzen.whosonfirst.export.flatfile(data_dir, debug=False)
		path = ff.export_feature(f)
	except Exception, e:
		error = "failed to export geojson, because %s" % e
		logging.error(error)
		return jsonify(ok=0, error=error)

	if not path:
		return jsonify(ok=0, error="File export returned an empty path, maybe check file permissions?")

	# Repeat back the file we just wrote
	try:
		gf = open(path, 'r')
		exported = gf.read()
	except Exception, e:
		error = "failed to read exported geojson, because %s" % e
		logging.error(error)
		return jsonify(ok=0, error=error)

	return jsonify(ok=1, geojson=exported)

@app.route('/pip', methods=['GET'])
def geojson_hierarchy():
	data_endpoint = 'https://whosonfirst.mapzen.com/data/'
	lat = float(request.args.get('latitude'))
	lng = float(request.args.get('longitude'))
	wof_id = request.args.get('wof_id')
	placetype = request.args.get('placetype')
	pip_server = mapzen.whosonfirst.pip.server(hostname='pip.mapzen.com', scheme='https', port=443)

	if (mapzen.whosonfirst.placetypes.is_valid_placetype(placetype) == False):
		return jsonify(ok=0, error="What is that placetype?")

	try:
		parents = mapzen.whosonfirst.pip.utils.get_reverse_geocoded(lat, lng, placetype, pip_server=pip_server)
	except Exception, e:
		error = "failed to determine parents, because %s" % e
		logging.error(error)
		return jsonify(ok=0, error=error)

	# Ok, here is a weird situation. Let's say you've got a venue out in the
	# desert. Its usual parents like neighbourhood or microhood don't exist out
	# here. But we still want to build a hierarchy if our first attempt comes
	# up empty. We're just gonna retry, with locality as the new placetype.
	# Eventually this will probably get handled in a Python library, but for now
	# we'll just do it the HULKSMASH way. (20170504/dphiffer)
	if placetype == 'venue' and len(parents) == 0:
		shim_placetype = 'locality'
		parents = mapzen.whosonfirst.pip.utils.get_reverse_geocoded(lat, lng, shim_placetype, pip_server=pip_server)

	try:
		hierarchy = mapzen.whosonfirst.pip.utils.get_hierarchy(parents, wof_id, placetype)
	except Exception, e:
		error = "failed to determine hierarchy, because %s" % e
		logging.error(error)
		return jsonify(ok=0, error=error)

	return jsonify(ok=1, hierarchy=hierarchy, parents=parents)

@app.route('/nearby', methods=['GET'])
def nearby():

	# Please put me in a config file
	api_key = "mapzen-Gvj9yGE"

	lat = float(request.args.get('latitude'))
	lng = float(request.args.get('longitude'))
	name = request.args.get('name')

	results = []
	result_ids = []

	method = 'whosonfirst.places.getByLatLon'
	params = { 'api_key': api_key, 'method': method, 'latitude': lat, 'longitude': lng, 'placetype': 'neighbourhood' }

	rsp = requests.get('https://whosonfirst-api.mapzen.com/', params=params)
	data = json.loads(rsp.content)

	if (data['places'] and len(data['places']) > 0):
		for neighbourhood in data['places']:

			wofid = neighbourhood['wof:id']
			method = 'whosonfirst.places.search'
			params = { 'api_key': api_key, 'method': method, 'neighbourhood_id': wofid, 'names': name, 'extras': 'addr:', 'placetype': 'venue' }

			rsp = requests.get('https://whosonfirst-api.mapzen.com/', params=params)
			data = json.loads(rsp.content)

			if data['places']:
				for pl in data['places']:
					if pl['wof:id'] not in result_ids:
						result_ids.append(pl['wof:id'])
						results.append(pl)

	method = "whosonfirst.places.getNearby"
	params = { 'api_key': api_key, 'method': method, 'latitude': lat, 'longitude': lng, 'placetype': 'venue', 'radius': 30, 'extras': 'addr:full' }

	rsp = requests.get('https://whosonfirst-api.mapzen.com/', params=params)
	print rsp.content
	data = json.loads(rsp.content)

	if (data['places'] and len(data['places']) > 0):
		for pl in data['places']:
			if pl['wof:id'] not in result_ids:
				result_ids.append(pl['wof:id'])
				results.append(pl)

	return jsonify(ok=1, results=results)

if __name__ == "__main__":
	import sys
	import optparse
	import ConfigParser

	opt_parser = optparse.OptionParser()

	opt_parser.add_option('-p', '--port', dest='port', action='store', default=8181, help='')
	opt_parser.add_option('-d', '--dir', dest='dir', action='store', default='/usr/local/mapzen/whosonfirst-www-boundaryissues/pending/', help='wof_pending_dir')
	opt_parser.add_option('-v', '--verbose', dest='verbose', action='store_true', default=False, help='Be chatty (default is false)')

	options, args = opt_parser.parse_args()

	if options.verbose:
		logging.basicConfig(level=logging.DEBUG)
	else:
		logging.basicConfig(level=logging.INFO)

	os.environ['WOF_PENDING_DIR'] = options.dir
	port = int(options.port)

	app.run(port=port)
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*
"""
File: report.py
Author: Arve Seljebu
Email: arve.seljebu@gmail.com
Github: arve0
Description: Generates a report on defferences between english and given wiki.
"""

import sys
from sources import Pages
from sources import Commands
from cache import Cache
from language import Language
import pywikibot

# utf8 hack
# http://stackoverflow.com/questions/492483/setting-the-correct-encoding-when-piping-stdout-in-python
import codecs
sys.stdout = codecs.getwriter('utf8')(sys.stdout)

if sys.version_info[0] != 2:
    print u'This program is written for python2, as pywikibot uses python2.'
    sys.exit()

def update_cache(language, namespace):
    """Update cache of given language and namespace."""
    suffix = '-' + language + '-' + namespace

    msg = u'Updating cache for {0}, namespace {1}'\
            .format(Language(language), namespace)
    print msg
    print u'='*len(msg)

    pages = Cache(Pages(namespace=namespace, language=language).get,
                  'pages' + suffix, force=True)
    commands = Cache(Commands(language, pages=pages.data).get,
                     'commands-' + language, force=True)

def find_missing(language, namespace, console_output=False):
    """Find missing pages (not translated from english or not added to wiki)."""
    suffix = '-' + language + '-' + namespace

    print u'Getting pages and commands from cache'

    pages = Cache(Pages(namespace=namespace, language=language).get,
                  'pages' + suffix)

    commands = Cache(Commands(language, pages=pages.data).get,
                     'commands-' + language)

    # Missing pages
    msg = u'== Missing command pages ==\n'
    if console_output:
        print msg

    msg += u'{| class="wikitable"\n'
    msg += u'|- <!-- header -->\n'
    msg += u'! Missing !! English page\n'

    row = 1
    cmd_string = u' ' + commands.data['Command']['translation']
    for (command_key, command) in commands.data.iteritems():
        if 'wikiid' not in command.keys():
            if console_output:
                print (u'Wikipage missing for command {0}'
                       .format(command['translation']))
            title = command['translation'] + cmd_string
            link = namespace + u':' + title
            en_title = command_key + u' Command'
            en_link = namespace + u':' + en_title
            msg += u'|- <-- row {0} -->\n'.format(row)
            row += 1
            msg += (u'| [[{0}|{1}]] || [[:en:{2}|{3}]]\n'
                    .format(link, title, en_link, en_title))

    msg += u'|}\n'

    return msg

def find_updated(language, namespace, console_output=False):
    """Find command pages which is updated in English wiki."""
    suffix = '-' + language + '-' + namespace

    print u'Getting pages and commands from cache'

    en_pages = Cache(Pages(namespace=namespace).get, 'pages-en-' + namespace)
    pages = Cache(Pages(namespace=namespace, language=language).get,
                  'pages' + suffix)

    en_commands = Cache(Commands(pages=en_pages.data).get, 'commands-en')
    commands = Cache(Commands(language, pages=pages.data).get,
                     'commands-' + language)

    # Updated pages
    msg = u'== Updated command pages ==\n'
    if console_output:
        print msg

    msg += u'{| class="wikitable"\n'
    msg += u'|- <!-- header -->\n'
    msg += u'! Page !! Last edit !! English page !! Last edit\n'

    row = 1
    for (command_name, command) in commands.data.iteritems():
        if 'wikiid' not in command.keys():
            # command does not exist in wiki
            continue

        page = [p for p in pages.data if p['id'] == command['wikiid']]
        en_command = en_commands.data[command_name]
        en_page = [p for p in en_pages.data if p['id'] == en_command['wikiid']]

        # make sure we get only one match
        assert len(page) == 1 and len(en_page) == 1
        page = page[0]
        en_page = en_page[0]

        # short names
        title = page['title']
        link = page['fullTitle']
        time = page['editTime']
        en_title = en_page['title']
        en_link = en_page['fullTitle']
        en_time = en_page['editTime']
        if time < en_time:
            # put time 50 chars to right
            if console_output:
                print u'{0} updated\r\x1b[50C{1}'.format(title, time)
                print u'{0} updated\r\x1b[50C{1}'.format(en_title, en_time)
                print u''
            msg += u'|- <-- row {0} -->\n'.format(row)
            row += 1
            msg += u'| [[{0}|{1}]] || {2}'.format(link, title, time)
            msg += (u'|| [[:en:{0}|{1}]] || {2}\n'
                    .format(en_link, en_title, en_time))

    msg += u'|}\n'

    return msg

def wiki(language, namespace):
    """
    Run all reports and write result to wikipage 'Translation Report'.
    """

    site = pywikibot.Site(code=language, fam='geogebra')
    page = pywikibot.Page(site, 'Translation Report')

    page.text = find_missing(language, namespace)
    page.text += find_updated(language, namespace)

    print (u'Saving to http://wiki.geogebra.org/{0}/Translation_Report'
           .format(language))
    page.save(comment='geogebra wiki translation helper')

def print_usage():
    """
    Print usage of this script.
    """

    usage = '''
    Usage
    =====
    {0} command language [namespace]

    Commands
    ========
    cache: \tUpdate cache of given language and namespace.

    missing: \tFind missing command wikipages.

    updated: \tFind wikipages with newer/updated English version.

    Default values
    ==============
    namespace: 'Manual'
'''.format(sys.argv[0])

    print usage

def main():
    """ process args """
    length = len(sys.argv)
    if length == 4:
        language = sys.argv[2].lower()
        namespace = sys.argv[3].capitalize()
    elif length == 3:
        language = sys.argv[2].lower()
        namespace = 'Manual'
    else:
        print_usage()
        sys.exit()

    cmd = sys.argv[1]
    if cmd == 'cache':
        update_cache(language, namespace)
    elif cmd == 'missing':
        find_missing(language, namespace, True)
    elif cmd == 'updated':
        find_updated(language, namespace, True)
    elif cmd == 'wiki':
        wiki(language, namespace)

if __name__ == '__main__':
    main()
<commit_msg>new feature: size difference<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*
"""
File: report.py
Author: Arve Seljebu
Email: arve.seljebu@gmail.com
Github: arve0
Description: Generates a report on defferences between english and given wiki.
"""

import sys
from sources import Pages
from sources import Commands
from cache import Cache
from language import Language
import pywikibot

# utf8 hack
# http://stackoverflow.com/questions/492483/setting-the-correct-encoding-when-piping-stdout-in-python
import codecs
sys.stdout = codecs.getwriter('utf8')(sys.stdout)

if sys.version_info[0] != 2:
    print u'This program is written for python2, as pywikibot uses python2.'
    sys.exit()

def print_usage():
    """
    Print usage of this script.
    """

    usage = '''
    Usage
    =====
    {0} command language [namespace]

    Commands
    ========
    cache: \tUpdate cache of given language and namespace.

    missing: \tFind missing command wikipages.

    size: \tFind wikipages with large size difference to English version.

    updated: \tFind wikipages with newer/updated English version.

    wiki: \tRun all reports and upload the report to wiki.

    Default values
    ==============
    namespace: 'Manual' - Should be in english. Eg: Main, Manual.
                          List here http://wiki.geogebra.org/en/Special:Categories
'''.format(sys.argv[0])

    print usage

def update_cache(language, namespace):
    """Update cache of given language and namespace."""
    suffix = '-' + language + '-' + namespace

    msg = u'Updating cache for {0}, namespace {1}'\
            .format(Language(language), namespace)
    print msg
    print u'='*len(msg)

    pages = Cache(Pages(namespace=namespace, language=language).get,
                  'pages' + suffix, force=True)
    commands = Cache(Commands(language, pages=pages.data).get,
                     'commands-' + language, force=True)

    # deletion invokes saving of cache
    del pages
    del commands

def find_missing(language, namespace, console_output=False):
    """Find missing pages (not translated from english or not added to wiki)."""
    suffix = '-' + language + '-' + namespace

    print u'Getting pages and commands from cache'

    pages = Cache(Pages(namespace=namespace, language=language).get,
                  'pages' + suffix)

    commands = Cache(Commands(language, pages=pages.data).get,
                     'commands-' + language)

    # Missing pages
    msg = u'== Missing command pages ==\n'
    if console_output:
        print msg

    msg += u'{| class="wikitable"\n'
    msg += u'|- <!-- header -->\n'
    msg += u'! Missing !! English page\n'

    row = 1
    cmd_string = u' ' + commands.data['Command']['translation']
    for (command_key, command) in commands.data.iteritems():
        if 'wikiid' not in command.keys():
            if console_output:
                print (u'Wikipage missing for command {0}'
                       .format(command['translation']))
            title = command['translation'] + cmd_string
            link = namespace + u':' + title
            en_title = command_key + u' Command'
            en_link = namespace + u':' + en_title
            msg += u'|- <-- row {0} -->\n'.format(row)
            row += 1
            msg += (u'| [[{0}|{1}]] || [[:en:{2}|{3}]]\n'
                    .format(link, title, en_link, en_title))

    msg += u'|}\n'

    return msg

def find_updated(language, namespace, console_output=False):
    """Find command pages which is updated in English wiki."""
    suffix = '-' + language + '-' + namespace

    print u'Getting pages and commands from cache'

    en_pages = Cache(Pages(namespace=namespace).get, 'pages-en-' + namespace)
    pages = Cache(Pages(namespace=namespace, language=language).get,
                  'pages' + suffix)

    en_commands = Cache(Commands(pages=en_pages.data).get, 'commands-en')
    commands = Cache(Commands(language, pages=pages.data).get,
                     'commands-' + language)

    # Updated pages
    msg = u'== Updated command pages ==\n'
    if console_output:
        print msg

    msg += u'{| class="wikitable"\n'
    msg += u'|- <!-- header -->\n'
    msg += u'! Page !! Last edit !! English page !! Last edit\n'

    row = 1
    for (command_name, command) in commands.data.iteritems():
        if 'wikiid' not in command.keys():
            # command does not exist in wiki
            continue

        page = [p for p in pages.data if p['id'] == command['wikiid']]
        en_command = en_commands.data[command_name]
        en_page = [p for p in en_pages.data if p['id'] == en_command['wikiid']]

        # make sure we get only one match
        assert len(page) == 1 and len(en_page) == 1
        page = page[0]
        en_page = en_page[0]

        # short names
        title = page['title']
        link = page['fullTitle']
        time = page['editTime']
        en_title = en_page['title']
        en_link = en_page['fullTitle']
        en_time = en_page['editTime']
        if time < en_time:
            if console_output:
                # \r\x1b[50C - 50 chars right from start of line
                print u'{0} updated\r\x1b[50C{1}'.format(title, time)
                print u'{0} updated\r\x1b[50C{1}'.format(en_title, en_time)
                print u''
            msg += u'|- <-- row {0} -->\n'.format(row)
            row += 1
            msg += u'| [[{0}|{1}]] || {2}'.format(link, title, time)
            msg += (u'|| [[:en:{0}|{1}]] || {2}\n'
                    .format(en_link, en_title, en_time))

    msg += u'|}\n'

    return msg

def find_size_difference(language, namespace, console_output=False):
    """Find pages with large size difference."""
    suffix = '-' + language + '-' + namespace

    print u'Getting pages and commands from cache'

    en_pages = Cache(Pages(namespace=namespace).get, 'pages-en-' + namespace)
    pages = Cache(Pages(namespace=namespace, language=language).get,
                  'pages' + suffix)

    en_commands = Cache(Commands(pages=en_pages.data).get, 'commands-en')
    commands = Cache(Commands(language, pages=pages.data).get,
                     'commands-' + language)

    # Updated pages
    msg = u'== Size differences ==\n'
    if console_output:
        print msg
        print 'Title, largest, english title, difference.\n'

    msg += u'{| class="wikitable"\n'
    msg += u'|- <!-- header -->\n'
    msg += u'! Page !! Largest !! English page !! Size difference\n'

    size_differences = []
    for (command_name, command) in commands.data.iteritems():
        if 'wikiid' not in command.keys():
            # command does not exist in wiki
            continue

        page = [p for p in pages.data if p['id'] == command['wikiid']]
        en_command = en_commands.data[command_name]
        en_page = [p for p in en_pages.data if p['id'] == en_command['wikiid']]

        # make sure we get only one match
        assert len(page) == 1 and len(en_page) == 1
        page = page[0]
        en_page = en_page[0]

        # short names
        title = page['title']
        link = page['fullTitle']
        size = len(page['text'])
        en_title = en_page['title']
        en_link = en_page['fullTitle']
        en_size = len(en_page['text'])
        difference = abs(size - en_size)
        if difference > 300:
            # ~ three sentences difference
            largest = '<--' if size > en_size else '-->'
            size_differences.append({
                'difference': difference,
                'largest': largest,
                'title': title,
                'link': link,
                'en_title': en_title,
                'en_link': en_link,
            })

    # sort after size, highest first
    size_differences.sort(key=lambda obj: obj['difference'], reverse=True)
    row = 1
    for obj in size_differences:
        if console_output:
            # \r\x1b[50C - 70 chars right from start of line
            print (u'{0} {1} {2}\r\x1b[70C{3} chars'
                   .format(obj['title'], obj['largest'], obj['en_title'],
                           obj['difference']))
            print u''
        msg += u'|- <-- row {0} -->\n'.format(row)
        row += 1
        msg += u'| [[{0}|{1}]]\n'.format(obj['link'], obj['title'])
        msg += u'| {0}\n'.format(obj['largest'])
        msg += u'| [[:en:{0}|{1}]]\n'.format(obj['en_link'], obj['en_title'])
        msg += u'| {0}\n'.format(obj['difference'])

    msg += u'|}\n'

    return msg

def wiki(language, namespace):
    """
    Run all reports and write result to wikipage 'Translation Report'.
    """

    site = pywikibot.Site(code=language, fam='geogebra')
    page = pywikibot.Page(site, 'Translation Report')

    page.text = find_missing(language, namespace)
    page.text += find_updated(language, namespace)
    page.text += find_size_difference(language, namespace)

    print (u'Saving to http://wiki.geogebra.org/{0}/Translation_Report'
           .format(language))
    page.save(comment='geogebra wiki translation helper')

def main():
    """ process args """
    length = len(sys.argv)
    if length == 4:
        language = sys.argv[2].lower()
        namespace = sys.argv[3].capitalize()
    elif length == 3:
        language = sys.argv[2].lower()
        namespace = 'Manual'
    else:
        print_usage()
        sys.exit()

    cmd = sys.argv[1]
    if cmd == 'cache':
        update_cache(language, namespace)
    elif cmd == 'missing':
        find_missing(language, namespace, True)
    elif cmd == 'updated':
        find_updated(language, namespace, True)
    elif cmd == 'size':
        find_size_difference(language, namespace, True)
    elif cmd == 'wiki':
        wiki(language, namespace)

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>#!/usr/bin/env python

# Framework acting as a client.
import roslib; roslib.load_manifest('car_sim')
import numpy
import sys
import subprocess
import time
from Tools import *

import rospy
from tf.transformations import euler_from_quaternion
from geometry_msgs.msg import *
from car_sim.srv import *
from car_sim.msg import *

# This class assumes that you have both ROS properly installed AND Mark Cutter's car simulator.
# If the internal flag is True, the script will call the proper command to initialize a car simulation in Mark's simulator in a seperate process.
#    Note that this proccess does not terminate correctly and will still be running even if the Framework is killed. I do not adive using it
# If you want to initialize this simulation seperately use the command "roslaunch car_control control.launch veh:=RC num:=01 sim:=true"
#
# The visualize flag determines whether Mark's simulation should visualize the simulation. Use this command to initialize the visualizer: "roslaunch raven_rviz raven_world.launch"
#
# Email me at gadgy@mit.edu if you have any questions, Elliott

class RC_Com():
    def __init__(self, internal=False):
        self.state = CarState(pose=Pose(position=Point(0.00001,0.00001,0.00001)))
	self.XMAX = 3/2.0
	self.XMIN = -3/2.0
	self.YMAX = 2/2.0
	self.YMIN =  -2/2.0
	if internal:
		subprocess.Popen("roslaunch car_control control.launch veh:=RC num:=01 sim:=true", shell=True)

    def Step(self, s,a,visualize=False):
        rospy.wait_for_service('RC01/run_step')
        try:
            step = rospy.ServiceProxy('RC01/run_step', RunStep)
	    omegaDes, turn = self.decodeAction(a)
	    #print omegaDes
	    #omegaDes, turn = 0.00001, 0.000001
	    #omegaDes, turn =1,-1
	    resp1 = step(self.state,0.2,2*omegaDes/0.035,turn,visualize) # 0.005
            ns = resp1.finalState
	    if ns.pose.position.x >= self.XMAX:
		ns.pose.position.x = self.XMAX
		ns.Vx = 0
	    elif ns.pose.position.x <= self.XMIN:
		ns.pose.position.x = self.XMIN
		ns.Vx = 0
	    if ns.pose.position.y >= self.YMAX:
		ns.pose.position.y = self.YMAX
		ns.Vy = 0
	    elif ns.pose.position.y <= self.YMIN:
		ns.pose.position.y = self.YMIN
		ns.Vy = 0
      	    self.state = ns
            speed = (ns.Vx**2 + ns.Vy**2)**0.5
	    quat_head = [ns.pose.orientation.x,ns.pose.orientation.y,ns.pose.orientation.z,ns.pose.orientation.w]
	    heading = euler_from_quaternion(quat_head)[2]
	    #print ns.pose.position.x, ns.pose.position.z
            new_state = numpy.array([ns.pose.position.x,ns.pose.position.y,speed,heading])
            #print new_state
            return new_state
        except rospy.ServiceException, e:
            print "Service call failed: %s"%e

    def resetState(self):
	self.state = CarState(pose=Pose(position=Point(0.00001,0.00001,0.00001)))

    def decodeAction(self, a):
        omegaDes,turn                = id2vec(a,[3,3])
        omegaDes                     -= 1
        turn                    -= 1
        return omegaDes+0.00000001, turn+0.00000001
<commit_msg>small ROS comment change<commit_after>#!/usr/bin/env python

# Framework acting as a client.
import roslib; roslib.load_manifest('car_sim')
import numpy
import sys
import subprocess
import time
from Tools import *

import rospy
from tf.transformations import euler_from_quaternion
from geometry_msgs.msg import *
from car_sim.srv import *
from car_sim.msg import *

# This class assumes that you have both ROS properly installed AND Mark Cutter's car simulator.
# If the internal flag is True, the script will call the proper command to initialize a car simulation in Mark's simulator in a seperate process.
#    Note that this proccess does not terminate correctly and will still be running even if the Framework is killed. I do not adive using it
# If you want to initialize this simulation seperately use the command "roslaunch car_control control.launch veh:=RC num:=01 sim:=true"
#
# The visualize flag determines whether Mark's simulation should visualize the simulation. Use this command to initialize the visualizer: "roslaunch raven_rviz raven_world.launch"
#
# See the following two tutorials to get ROS communicating between multiple computers:
# http://www.ros.org/wiki/ROS/NetworkSetup
# http://www.ros.org/wiki/ROS/Tutorials/MultipleMachines
#
# Email me at gadgy@mit.edu if you have any questions, Elliott

class RC_Com():
    def __init__(self, internal=False):
        self.state = CarState(pose=Pose(position=Point(0.00001,0.00001,0.00001)))
	self.XMAX = 3/2.0
	self.XMIN = -3/2.0
	self.YMAX = 2/2.0
	self.YMIN =  -2/2.0
	if internal:
		subprocess.Popen("roslaunch car_control control.launch veh:=RC num:=01 sim:=true", shell=True)

    def Step(self, s,a,visualize=False):
        rospy.wait_for_service('RC01/run_step')
        try:
            step = rospy.ServiceProxy('RC01/run_step', RunStep)
	    omegaDes, turn = self.decodeAction(a)
	    #print omegaDes
	    #omegaDes, turn = 0.00001, 0.000001
	    #omegaDes, turn =1,-1
	    resp1 = step(self.state,0.2,2*omegaDes/0.035,turn,visualize) # 0.005
            ns = resp1.finalState
	    if ns.pose.position.x >= self.XMAX:
		ns.pose.position.x = self.XMAX
		ns.Vx = 0
	    elif ns.pose.position.x <= self.XMIN:
		ns.pose.position.x = self.XMIN
		ns.Vx = 0
	    if ns.pose.position.y >= self.YMAX:
		ns.pose.position.y = self.YMAX
		ns.Vy = 0
	    elif ns.pose.position.y <= self.YMIN:
		ns.pose.position.y = self.YMIN
		ns.Vy = 0
      	    self.state = ns
            speed = (ns.Vx**2 + ns.Vy**2)**0.5
	    quat_head = [ns.pose.orientation.x,ns.pose.orientation.y,ns.pose.orientation.z,ns.pose.orientation.w]
	    heading = euler_from_quaternion(quat_head)[2]
	    #print ns.pose.position.x, ns.pose.position.z
            new_state = numpy.array([ns.pose.position.x,ns.pose.position.y,speed,heading])
            #print new_state
            return new_state
        except rospy.ServiceException, e:
            print "Service call failed: %s"%e

    def resetState(self):
	self.state = CarState(pose=Pose(position=Point(0.00001,0.00001,0.00001)))

    def decodeAction(self, a):
        omegaDes,turn                = id2vec(a,[3,3])
        omegaDes                     -= 1
        turn                    -= 1
        return omegaDes+0.00000001, turn+0.00000001
<|endoftext|>


<commit_before># This file is part of colorspacious
# Copyright (C) 2014 Nathaniel Smith <njs@pobox.com>
# See file LICENSE.txt for license information.

# This file must be kept very simple, because it is consumed from several
# places -- it is imported by colorspacious/__init__.py, execfile'd by
# setup.py, etc.

# We use a simple scheme:
#   1.0.0 -> 1.0.0-dev -> 1.1.0 -> 1.1.0-dev
# where the -dev versions are never released into the wild, they're just what
# we stick into the VCS in between releases.
#
# This is compatible with PEP 440:
#   http://legacy.python.org/dev/peps/pep-0440/
# via the use of the "local suffix" "+dev", which is disallowed on index
# servers and causes 1.0.0+dev to sort after plain 1.0.0, which is what we
# want. (Contrast with the special suffix 1.0.0.dev, which sorts *before*
# 1.0.0.)

__version__ = "0.1.0+dev"
<commit_msg>Bump version to 1.0.0<commit_after># This file is part of colorspacious
# Copyright (C) 2014 Nathaniel Smith <njs@pobox.com>
# See file LICENSE.txt for license information.

# This file must be kept very simple, because it is consumed from several
# places -- it is imported by colorspacious/__init__.py, execfile'd by
# setup.py, etc.

# We use a simple scheme:
#   1.0.0 -> 1.0.0+dev -> 1.1.0 -> 1.1.0+dev
# where the +dev versions are never released into the wild, they're just what
# we stick into the VCS in between releases.
#
# This is compatible with PEP 440:
#   http://legacy.python.org/dev/peps/pep-0440/
# via the use of the "local suffix" "+dev", which is disallowed on index
# servers and causes 1.0.0+dev to sort after plain 1.0.0, which is what we
# want. (Contrast with the special suffix 1.0.0.dev, which sorts *before*
# 1.0.0.)

__version__ = "1.0.0"
<|endoftext|>


<commit_before>##########################################################################
#          (C) Vrije Universiteit, Amsterdam (the Netherlands)            #
#                                                                         #
# This file is part of AmCAT - The Amsterdam Content Analysis Toolkit     #
#                                                                         #
# AmCAT is free software: you can redistribute it and/or modify it under  #
# the terms of the GNU Affero General Public License as published by the  #
# Free Software Foundation, either version 3 of the License, or (at your  #
# option) any later version.                                              #
#                                                                         #
# AmCAT is distributed in the hope that it will be useful, but WITHOUT    #
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or   #
# FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public     #
# License for more details.                                               #
#                                                                         #
# You should have received a copy of the GNU Affero General Public        #
# License along with AmCAT.  If not, see <http://www.gnu.org/licenses/>.  #
###########################################################################
import datetime
import pickle
import zlib
import re

import dateutil.parser
import functools
import hashlib

from amcat.models import Project, ArticleSet, TaskHandler, CodingJob
from amcat.models.authorisation import ROLE_PROJECT_METAREADER
from amcat.scripts.forms import SelectionForm
from amcat.tools.caching import cached
from amcat.tools.progress import ProgressMonitor
from django import forms
from django.contrib.auth.models import User
from django.contrib.humanize.templatetags.humanize import naturaltime
from django.core.cache import cache
from django.core.exceptions import ValidationError, PermissionDenied
from django.core.urlresolvers import reverse
from django.http import QueryDict, HttpResponse
from navigator.views.scriptview import CeleryProgressUpdater
from settings import SECRET_KEY

DOWNLOAD_HEADER = "Content-Disposition: attachment; "
RE_CAPITALS = re.compile("([A-Z])")

def to_querydict(dict):
    """
    Converts a value created by querydict_dict back into a Django QueryDict value.
    """
    q = QueryDict("", mutable=True)
    for k, v in dict.items():
        q.setlist(k, v)
    q._mutable = False
    return q

class QueryActionForm(SelectionForm):
    output_type = forms.ChoiceField(choices=())
    download = forms.BooleanField(initial=False, required=False)

    def __init__(self, user, *args, **kwargs):
        super(QueryActionForm, self).__init__(*args, **kwargs)
        self.user = user

class QueryActionHandler(TaskHandler):
    @classmethod
    def serialise_arguments(cls, arguments):
        arguments = super(QueryActionHandler, cls).serialise_arguments(arguments)
        user = arguments['user']
        arguments['user'] = None if user.is_anonymous() else user.id
        arguments['project'] = arguments['project'].id
        arguments['articlesets'] = [aset.id for aset in arguments['articlesets']]
        arguments['codingjobs'] = [cj.id for cj in arguments.get('codingjobs', [])]

        if arguments['data'] is not None and isinstance(arguments["data"], QueryDict):
            arguments['data'] = dict(arguments['data'].lists())

        return arguments

    @classmethod
    def deserialise_arguments(cls, arguments):
        arguments = super(QueryActionHandler, cls).deserialise_arguments(arguments)
        arguments['user'] = None if arguments['user'] is None else User.objects.get(id=arguments['user'])
        arguments['project'] = Project.objects.get(id=arguments['project'])
        arguments['articlesets'] = ArticleSet.objects.filter(id__in=arguments['articlesets'])
        arguments['codingjobs'] = CodingJob.objects.filter(id__in=arguments.get('codingjobs', []))

        if arguments['data'] is not None:
            arguments['data'] = to_querydict(arguments['data'])

        return arguments

    @cached
    def get_query_action(self):
        query_action = self.task.get_class()(**self.task.get_arguments())
        query_action.get_form().full_clean()
        return query_action

    def run_task(self):
        query_action = self.get_query_action()
        updater = CeleryProgressUpdater(str(self.task.uuid))
        query_action.monitor.add_listener(updater.update)

        msg = "Task started, running {query_action.__class__.__name__}..".format(**locals())
        query_action.monitor.update(0, msg)

        try:
            form = query_action.get_form()
            query_action.before_run(form)
            result = query_action.run(form)
            cache_key = query_action.get_cache_key() if query_action.cache_hit else None
            return cache_key, result
        except Exception as e:
            raise

    def _get_content_type(self):
        """Returns content type of selected 'output_type'. This usually is a mimetype
        formatted like:

            application/json
            text/csv

        If clients want to add a specific 'meaning' to the data, the can add it using
        the separator comma:

            application/json,normalise
            text/csv,sparse

        In the latter cases only the part before the comma will be send back to the client
        as Content-Type header.
        """
        form = self.get_query_action().get_form()
        content_type = form.cleaned_data["output_type"]
        content_type = content_type.split(";")[0]
        return content_type

    def get_response(self):
        cache_key, result = self.get_result()
        response = HttpResponse(content=result)
        response["X-Query-Cache-Hit"] = int(cache_key is not None)
        if cache_key:
            response["X-Query-Cache-Key"] = cache_key
            timestamp = cache.get("{}.timestamp".format(cache_key))
            timestamp = dateutil.parser.parse(timestamp)
            response["X-Query-Cache-Timestamp"] = timestamp.isoformat()
            response["X-Query-Cache-Natural-Timestamp"] = naturaltime(timestamp)
        response["Content-Disposition"] = "attachment"
        response["Content-Type"] = self._get_content_type()
        return response

    def get_redirect(self):
        return reverse("api:queryaction-index"), "No redirect"

class NotInCacheError(Exception):
    pass

def is_valid_cache_key(cache_key):
    return len(cache_key) == 64 + 12 and cache_key.endswith(".query-cache")

class QueryAction(object):
    form_class = QueryActionForm
    task_handler = QueryActionHandler
    output_types = None
    required_role = ROLE_PROJECT_METAREADER
    ignore_cache_fields = ("output_type",)
    monitor_steps = None

    def __init__(self, user, project, api_host, articlesets, codingjobs=None, data=None):
        """
        @type project: amcat.models.Project
        @type articlesets: django.db.QuerySet
        @type data: NoneType, django.http.QueryDict
        """
        self.user = user
        self.project = project
        self.api_host = api_host
        self.articlesets = articlesets
        self.codingjobs = codingjobs
        self.data = data
        self.cache_hit = False

        if self.monitor_steps is None:
            self.monitor = ProgressMonitor()
        else:
            self.monitor = ProgressMonitor(total=self.monitor_steps)

        assert(issubclass(self.form_class, SelectionForm))

    @functools.lru_cache()
    def get_cache_key(self) -> str:
        """Returns a cache key (SHA256 hexdigest) of user id and form hash. To prevent guessing
        attacks, we also embed the Django SECRET_KEY (240 random bits in AmCAT)."""
        form_hash = self.get_form().get_hash(ignore_fields=self.ignore_cache_fields)
        query_hash = hashlib.sha256("{}|{}|{}".format(SECRET_KEY, self.user.id, form_hash).encode("ascii")).hexdigest()
        return "{}.query-cache".format(query_hash)

    def serialize_cache_value(self, value):
        """Pickle then compress value"""
        return zlib.compress(pickle.dumps(value))

    def desearialize_cache_value(self, value):
        """Decompress then depickle value"""
        return pickle.loads(zlib.decompress(value))

    def get_cache(self):
        """Get cached value for this particular form+user. Raises NotInCacheError if not cached
        value is found."""
        cache_key = self.get_cache_key()
        value = cache.get(cache_key)
        if value is None:
            raise NotInCacheError("Cache value for {} was not found".format(cache_key))
        self.cache_hit = True
        return self.desearialize_cache_value(value)

    def set_cache(self, value):
        """Sets cache for this particular form+user to 'value'. The value is serialized
        with QueryAction.serialize_cache_value. Note that large values may not fit in
        memcached."""
        timestamp = datetime.datetime.now().isoformat()
        cache.set("{}.timestamp".format(self.get_cache_key()), timestamp, timeout=7200+1)
        cache.set(self.get_cache_key(), self.serialize_cache_value(value), timeout=7200)

    def get_form_kwargs(self, **kwargs):
        return dict({
            "data": self.data, "project": self.project,
            "articlesets": self.articlesets, "user": self.user,
            "codingjobs": self.codingjobs
        }, **kwargs)

    def get_form_class(self):
        return self.form_class

    def get_task_handler(self):
        return self.task_handler

    def target_project(self, form):
        return self.project

    @cached
    def get_form(self) -> SelectionForm:
        form = self.form_class(**self.get_form_kwargs())
        form.fields["output_type"].choices = self.output_types

        if len(self.output_types) <= 1:
            form.fields["output_type"].widget.attrs["type"] = "hidden"

        return form

    def before_run(self, form):
        self.check_permission(form)

    def run(self, form):
        """Needs to be overriden by subclass. Return value must be at least
        json serialisable, preferably bytes.

        @param form: cleaned form
        @type form: (subclass of) QueryActionForm"""
        raise NotImplementedError

    def run_delayed(self):
        """
        Put this task in celery queue. Returns a task handler.

        @raises: django.core.exceptions.ValidationError if form invalid
        @rtype: QueryActionHandler
        """
        form = self.get_form()

        if not form.is_valid():
            raise ValidationError(form._errors)
        self.check_permission(form)

        return self.get_task_handler().call(
            target_class=self.__class__, user=self.user,
            project=self.project, arguments={
                "user": self.user, "project": self.project, "api_host": self.api_host,
                "data": self.data, "articlesets": self.articlesets,
                "codingjobs": self.codingjobs
            }
        )

    def check_permission(self, form):
        project = self.target_project(form)
        actual = project.get_role_id(self.user)
        if actual < self.required_role:
            raise PermissionDenied("User has no permission to perform {self.__class__.__name__}"
                                   " on project {project}".format(**locals()))

    @classmethod
    def get_action_name(cls):
        if hasattr(cls, "action_name"):
            return cls.action_name
        name = cls.__name__
        if name.endswith("Action"):
            name = name[:-6]
        return name.lower()

    @classmethod
    def get_action_label(cls):
        if hasattr(cls, "action_label"):
            return cls.action_label
        name = cls.__name__
        if name.endswith("Action"):
            name = name[:-6]
        return RE_CAPITALS.sub(" \\1", name).strip()
<commit_msg>Made query api-host kwarg for better compatibility<commit_after>##########################################################################
#          (C) Vrije Universiteit, Amsterdam (the Netherlands)            #
#                                                                         #
# This file is part of AmCAT - The Amsterdam Content Analysis Toolkit     #
#                                                                         #
# AmCAT is free software: you can redistribute it and/or modify it under  #
# the terms of the GNU Affero General Public License as published by the  #
# Free Software Foundation, either version 3 of the License, or (at your  #
# option) any later version.                                              #
#                                                                         #
# AmCAT is distributed in the hope that it will be useful, but WITHOUT    #
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or   #
# FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public     #
# License for more details.                                               #
#                                                                         #
# You should have received a copy of the GNU Affero General Public        #
# License along with AmCAT.  If not, see <http://www.gnu.org/licenses/>.  #
###########################################################################
import datetime
import pickle
import zlib
import re

import dateutil.parser
import functools
import hashlib

from amcat.models import Project, ArticleSet, TaskHandler, CodingJob
from amcat.models.authorisation import ROLE_PROJECT_METAREADER
from amcat.scripts.forms import SelectionForm
from amcat.tools.caching import cached
from amcat.tools.progress import ProgressMonitor
from django import forms
from django.contrib.auth.models import User
from django.contrib.humanize.templatetags.humanize import naturaltime
from django.core.cache import cache
from django.core.exceptions import ValidationError, PermissionDenied
from django.core.urlresolvers import reverse
from django.http import QueryDict, HttpResponse
from navigator.views.scriptview import CeleryProgressUpdater
from settings import SECRET_KEY

DOWNLOAD_HEADER = "Content-Disposition: attachment; "
RE_CAPITALS = re.compile("([A-Z])")

def to_querydict(dict):
    """
    Converts a value created by querydict_dict back into a Django QueryDict value.
    """
    q = QueryDict("", mutable=True)
    for k, v in dict.items():
        q.setlist(k, v)
    q._mutable = False
    return q

class QueryActionForm(SelectionForm):
    output_type = forms.ChoiceField(choices=())
    download = forms.BooleanField(initial=False, required=False)

    def __init__(self, user, *args, **kwargs):
        super(QueryActionForm, self).__init__(*args, **kwargs)
        self.user = user

class QueryActionHandler(TaskHandler):
    @classmethod
    def serialise_arguments(cls, arguments):
        arguments = super(QueryActionHandler, cls).serialise_arguments(arguments)
        user = arguments['user']
        arguments['user'] = None if user.is_anonymous() else user.id
        arguments['project'] = arguments['project'].id
        arguments['articlesets'] = [aset.id for aset in arguments['articlesets']]
        arguments['codingjobs'] = [cj.id for cj in arguments.get('codingjobs', [])]

        if arguments['data'] is not None and isinstance(arguments["data"], QueryDict):
            arguments['data'] = dict(arguments['data'].lists())

        return arguments

    @classmethod
    def deserialise_arguments(cls, arguments):
        arguments = super(QueryActionHandler, cls).deserialise_arguments(arguments)
        arguments['user'] = None if arguments['user'] is None else User.objects.get(id=arguments['user'])
        arguments['project'] = Project.objects.get(id=arguments['project'])
        arguments['articlesets'] = ArticleSet.objects.filter(id__in=arguments['articlesets'])
        arguments['codingjobs'] = CodingJob.objects.filter(id__in=arguments.get('codingjobs', []))

        if arguments['data'] is not None:
            arguments['data'] = to_querydict(arguments['data'])

        return arguments

    @cached
    def get_query_action(self):
        query_action = self.task.get_class()(**self.task.get_arguments())
        query_action.get_form().full_clean()
        return query_action

    def run_task(self):
        query_action = self.get_query_action()
        updater = CeleryProgressUpdater(str(self.task.uuid))
        query_action.monitor.add_listener(updater.update)

        msg = "Task started, running {query_action.__class__.__name__}..".format(**locals())
        query_action.monitor.update(0, msg)

        try:
            form = query_action.get_form()
            query_action.before_run(form)
            result = query_action.run(form)
            cache_key = query_action.get_cache_key() if query_action.cache_hit else None
            return cache_key, result
        except Exception as e:
            raise

    def _get_content_type(self):
        """Returns content type of selected 'output_type'. This usually is a mimetype
        formatted like:

            application/json
            text/csv

        If clients want to add a specific 'meaning' to the data, the can add it using
        the separator comma:

            application/json,normalise
            text/csv,sparse

        In the latter cases only the part before the comma will be send back to the client
        as Content-Type header.
        """
        form = self.get_query_action().get_form()
        content_type = form.cleaned_data["output_type"]
        content_type = content_type.split(";")[0]
        return content_type

    def get_response(self):
        cache_key, result = self.get_result()
        response = HttpResponse(content=result)
        response["X-Query-Cache-Hit"] = int(cache_key is not None)
        if cache_key:
            response["X-Query-Cache-Key"] = cache_key
            timestamp = cache.get("{}.timestamp".format(cache_key))
            timestamp = dateutil.parser.parse(timestamp)
            response["X-Query-Cache-Timestamp"] = timestamp.isoformat()
            response["X-Query-Cache-Natural-Timestamp"] = naturaltime(timestamp)
        response["Content-Disposition"] = "attachment"
        response["Content-Type"] = self._get_content_type()
        return response

    def get_redirect(self):
        return reverse("api:queryaction-index"), "No redirect"

class NotInCacheError(Exception):
    pass

def is_valid_cache_key(cache_key):
    return len(cache_key) == 64 + 12 and cache_key.endswith(".query-cache")

class QueryAction(object):
    form_class = QueryActionForm
    task_handler = QueryActionHandler
    output_types = None
    required_role = ROLE_PROJECT_METAREADER
    ignore_cache_fields = ("output_type",)
    monitor_steps = None

    def __init__(self, user, project, articlesets, codingjobs=None, data=None, api_host=None):
        """
        @type project: amcat.models.Project
        @type articlesets: django.db.QuerySet
        @type data: NoneType, django.http.QueryDict
        """
        self.user = user
        self.project = project
        self.api_host = api_host
        self.articlesets = articlesets
        self.codingjobs = codingjobs
        self.data = data
        self.cache_hit = False

        if self.monitor_steps is None:
            self.monitor = ProgressMonitor()
        else:
            self.monitor = ProgressMonitor(total=self.monitor_steps)

        assert(issubclass(self.form_class, SelectionForm))

    @functools.lru_cache()
    def get_cache_key(self) -> str:
        """Returns a cache key (SHA256 hexdigest) of user id and form hash. To prevent guessing
        attacks, we also embed the Django SECRET_KEY (240 random bits in AmCAT)."""
        form_hash = self.get_form().get_hash(ignore_fields=self.ignore_cache_fields)
        query_hash = hashlib.sha256("{}|{}|{}".format(SECRET_KEY, self.user.id, form_hash).encode("ascii")).hexdigest()
        return "{}.query-cache".format(query_hash)

    def serialize_cache_value(self, value):
        """Pickle then compress value"""
        return zlib.compress(pickle.dumps(value))

    def desearialize_cache_value(self, value):
        """Decompress then depickle value"""
        return pickle.loads(zlib.decompress(value))

    def get_cache(self):
        """Get cached value for this particular form+user. Raises NotInCacheError if not cached
        value is found."""
        cache_key = self.get_cache_key()
        value = cache.get(cache_key)
        if value is None:
            raise NotInCacheError("Cache value for {} was not found".format(cache_key))
        self.cache_hit = True
        return self.desearialize_cache_value(value)

    def set_cache(self, value):
        """Sets cache for this particular form+user to 'value'. The value is serialized
        with QueryAction.serialize_cache_value. Note that large values may not fit in
        memcached."""
        timestamp = datetime.datetime.now().isoformat()
        cache.set("{}.timestamp".format(self.get_cache_key()), timestamp, timeout=7200+1)
        cache.set(self.get_cache_key(), self.serialize_cache_value(value), timeout=7200)

    def get_form_kwargs(self, **kwargs):
        return dict({
            "data": self.data, "project": self.project,
            "articlesets": self.articlesets, "user": self.user,
            "codingjobs": self.codingjobs
        }, **kwargs)

    def get_form_class(self):
        return self.form_class

    def get_task_handler(self):
        return self.task_handler

    def target_project(self, form):
        return self.project

    @cached
    def get_form(self) -> SelectionForm:
        form = self.form_class(**self.get_form_kwargs())
        form.fields["output_type"].choices = self.output_types

        if len(self.output_types) <= 1:
            form.fields["output_type"].widget.attrs["type"] = "hidden"

        return form

    def before_run(self, form):
        self.check_permission(form)

    def run(self, form):
        """Needs to be overriden by subclass. Return value must be at least
        json serialisable, preferably bytes.

        @param form: cleaned form
        @type form: (subclass of) QueryActionForm"""
        raise NotImplementedError

    def run_delayed(self):
        """
        Put this task in celery queue. Returns a task handler.

        @raises: django.core.exceptions.ValidationError if form invalid
        @rtype: QueryActionHandler
        """
        form = self.get_form()

        if not form.is_valid():
            raise ValidationError(form._errors)
        self.check_permission(form)

        return self.get_task_handler().call(
            target_class=self.__class__, user=self.user,
            project=self.project, arguments={
                "user": self.user, "project": self.project, "api_host": self.api_host,
                "data": self.data, "articlesets": self.articlesets,
                "codingjobs": self.codingjobs
            }
        )

    def check_permission(self, form):
        project = self.target_project(form)
        actual = project.get_role_id(self.user)
        if actual < self.required_role:
            raise PermissionDenied("User has no permission to perform {self.__class__.__name__}"
                                   " on project {project}".format(**locals()))

    @classmethod
    def get_action_name(cls):
        if hasattr(cls, "action_name"):
            return cls.action_name
        name = cls.__name__
        if name.endswith("Action"):
            name = name[:-6]
        return name.lower()

    @classmethod
    def get_action_label(cls):
        if hasattr(cls, "action_label"):
            return cls.action_label
        name = cls.__name__
        if name.endswith("Action"):
            name = name[:-6]
        return RE_CAPITALS.sub(" \\1", name).strip()
<|endoftext|>


<commit_before>import base64
from imaplib import IMAP4
from poplib import POP3
from smtplib import SMTP
from typing import Union
from urllib.parse import quote

import frappe
from frappe.integrations.google_oauth import GoogleOAuth
from frappe.utils import get_request_site_address

class OAuthenticationError(Exception):
	pass

class Oauth:
	def __init__(
		self,
		conn: Union[IMAP4, POP3, SMTP],
		email_account: str,
		email: str,
		access_token: str,
		refresh_token: str,
		service: str,
		mechanism: str = "XOAUTH2",
	) -> None:

		self.email_account = email_account
		self.email = email
		self.service = service
		self._mechanism = mechanism
		self._conn = conn
		self._access_token = access_token
		self._refresh_token = refresh_token

		self._validate()

	def _validate(self) -> None:
		if self.service != "GMail":
			raise NotImplementedError(
				f"Service {self.service} currently doesn't have oauth implementation."
			)

		if not self._refresh_token:
			frappe.throw(
				frappe._("Please Authorize OAuth."),
				OAuthenticationError,
				frappe._("OAuth Error"),
			)

	@property
	def _auth_string(self) -> str:
		return "user=%s\1auth=Bearer %s\1\1" % (self.email, self._access_token)

	def connect(self, _retry: int = 0) -> None:
		"""Connection method with retry on exception for Oauth"""
		try:
			if isinstance(self._conn, POP3):
				res = self._connect_pop()

				if not res.startswith(b"+OK"):
					raise

			elif isinstance(self._conn, IMAP4):
				self._connect_imap()

			else:
				# SMTP
				self._connect_smtp()

		except Exception:
			# maybe the access token expired - refreshing
			access_token = self._refresh_access_token()

			if not access_token or _retry > 0:
				frappe.throw(
					frappe._("Authentication Failed. Please Check and Update the credentials."),
					OAuthenticationError,
					frappe._("OAuth Error"),
				)

			self._access_token = access_token
			self.connect(_retry + 1)

	def _connect_pop(self) -> bytes:
		# poplib doesn't have AUTH command implementation
		res = self._conn._shortcmd(
			"AUTH {0} {1}".format(
				self._mechanism, base64.b64encode(bytes(self._auth_string, "utf-8")).decode("utf-8")
			)
		)

		return res

	def _connect_imap(self) -> None:
		self._conn.authenticate(self._mechanism, lambda x: self._auth_string)

	def _connect_smtp(self) -> None:
		self._conn.auth(self._mechanism, lambda x: self._auth_string, initial_response_ok=False)

	def _refresh_access_token(self) -> str:
		"""Refreshes access token via calling `refresh_access_token` method of oauth service object"""
		service_obj = self._get_service_object()
		access_token = service_obj.refresh_access_token(self._refresh_token).get("access_token", None)

		# set the new access token in db
		frappe.db.set_value(
			"Email Account", self.email_account, "access_token", access_token, update_modified=True
		)
		frappe.db.commit()
		return access_token

	def _get_service_object(self):
		"""Get Oauth service object"""

		return {
			"GMail": GoogleOAuth("mail", validate=False),
		}[self.service]

@frappe.whitelist(methods=["POST"])
def oauth_access(email_account: str, service: str = None):
	"""Used as a default endpoint/caller for all oauth services.
	Returns authorization url for redirection"""

	if not service:
		frappe.throw(frappe._("No Service is selected. Please select one and try again!"))

	doctype = "Email Account"

	# NOTE: setting this here, since we redirect to the service's auth page,
	# we lose the use_oauth value in the emal account form
	frappe.db.set_value(doctype, email_account, "use_oauth", 1, update_modified=False)

	if service == "GMail":
		return authorize_google_access(email_account, doctype)

def authorize_google_access(email_account, doctype: str = "Email Account", code: str = None):
	"""Facilitates google oauth for email.
	This is invoked 2 times - first time when user clicks `Authorze API Access` for getting the authorization url
	and second time for setting the refresh and access token in db when google redirects back with oauth code."""

	oauth_obj = GoogleOAuth("mail")

	if not code:
		return oauth_obj.get_authentication_url(
			get_request_site_address(True),
			state={
				"method": "frappe.email.oauth.authorize_google_access",
				"redirect": "/app/Form/{0}/{1}".format(quote(doctype), quote(email_account)),
				"email_account": email_account,
			},
		)

	res = oauth_obj.authorize(code, get_request_site_address(True))
	frappe.db.set_value(
		doctype,
		email_account,
		{"refresh_token": res.get("refresh_token"), "access_token": res.get("access_token")},
		update_modified=False,
	)
<commit_msg>fix: remove commit<commit_after>import base64
from imaplib import IMAP4
from poplib import POP3
from smtplib import SMTP
from typing import Union
from urllib.parse import quote

import frappe
from frappe.integrations.google_oauth import GoogleOAuth
from frappe.utils import get_request_site_address

class OAuthenticationError(Exception):
	pass

class Oauth:
	def __init__(
		self,
		conn: Union[IMAP4, POP3, SMTP],
		email_account: str,
		email: str,
		access_token: str,
		refresh_token: str,
		service: str,
		mechanism: str = "XOAUTH2",
	) -> None:

		self.email_account = email_account
		self.email = email
		self.service = service
		self._mechanism = mechanism
		self._conn = conn
		self._access_token = access_token
		self._refresh_token = refresh_token

		self._validate()

	def _validate(self) -> None:
		if not self._refresh_token:
			frappe.throw(
				frappe._("Please Authorize OAuth."),
				OAuthenticationError,
				frappe._("OAuth Error"),
			)

	@property
	def _auth_string(self) -> str:
		return f"user={self.email}\1auth=Bearer {self._access_token}\1\1"

	def connect(self, _retry: int = 0) -> None:
		"""Connection method with retry on exception for Oauth"""
		try:
			if isinstance(self._conn, POP3):
				res = self._connect_pop()

				if not res.startswith(b"+OK"):
					raise

			elif isinstance(self._conn, IMAP4):
				self._connect_imap()

			else:
				# SMTP
				self._connect_smtp()

		except Exception:
			# maybe the access token expired - refreshing
			access_token = self._refresh_access_token()

			if not access_token or _retry > 0:
				frappe.throw(
					frappe._("Authentication Failed. Please Check and Update the credentials."),
					OAuthenticationError,
					frappe._("OAuth Error"),
				)

			self._access_token = access_token
			self.connect(_retry + 1)

	def _connect_pop(self) -> bytes:
		# poplib doesn't have AUTH command implementation
		res = self._conn._shortcmd(
			"AUTH {0} {1}".format(
				self._mechanism, base64.b64encode(bytes(self._auth_string, "utf-8")).decode("utf-8")
			)
		)

		return res

	def _connect_imap(self) -> None:
		self._conn.authenticate(self._mechanism, lambda x: self._auth_string)

	def _connect_smtp(self) -> None:
		self._conn.auth(self._mechanism, lambda x: self._auth_string, initial_response_ok=False)

	def _refresh_access_token(self) -> str:
		"""Refreshes access token via calling `refresh_access_token` method of oauth service object"""
		service_obj = self._get_service_object()
		access_token = service_obj.refresh_access_token(self._refresh_token).get("access_token", None)

		# set the new access token in db
		frappe.db.set_value(
			"Email Account", self.email_account, "access_token", access_token, update_modified=False
		)
		return access_token

	def _get_service_object(self):
		"""Get Oauth service object"""

		return {
			"GMail": GoogleOAuth("mail", validate=False),
		}[self.service]

@frappe.whitelist(methods=["POST"])
def oauth_access(email_account: str, service: str):
	"""Used as a default endpoint/caller for all oauth services.
	Returns authorization url for redirection"""

	if not service:
		frappe.throw(frappe._("No Service is selected. Please select one and try again!"))

	doctype = "Email Account"

	# NOTE: setting this here, since we redirect to the service's auth page,
	# we lose the use_oauth value in the emal account form
	frappe.db.set_value(doctype, email_account, "use_oauth", 1, update_modified=False)

	if service == "GMail":
		return authorize_google_access(email_account, doctype)

	raise NotImplementedError(f"Service {service} currently doesn't have oauth implementation.")

def authorize_google_access(email_account, doctype: str = "Email Account", code: str = None):
	"""Facilitates google oauth for email.
	This is invoked 2 times - first time when user clicks `Authorze API Access` for getting the authorization url
	and second time for setting the refresh and access token in db when google redirects back with oauth code."""

	oauth_obj = GoogleOAuth("mail")

	if not code:
		return oauth_obj.get_authentication_url(
			get_request_site_address(True),
			state={
				"method": "frappe.email.oauth.authorize_google_access",
				"redirect": "/app/Form/{0}/{1}".format(quote(doctype), quote(email_account)),
				"email_account": email_account,
			},
		)

	res = oauth_obj.authorize(code, get_request_site_address(True))
	frappe.db.set_value(
		doctype,
		email_account,
		{"refresh_token": res.get("refresh_token"), "access_token": res.get("access_token")},
		update_modified=False,
	)
<|endoftext|>


<commit_before>"""
age.py -- show age of code
"""
# pylint: disable=W0640

# list files in Git branch
# git ls-tree --name-status --full-tree -r v4.0.0

import datetime
import math
import os
import re

import matplotlib.pyplot as plt
from django.core.management.base import BaseCommand
from git import Repo
from palettable import colorbrewer
from PIL import Image, ImageColor, ImageDraw

def get_tags(repo):
    tags = [tag.name.lstrip('v') for tag in repo.tags
        if tag.name.startswith('v3.') or tag.name == 'v4.0.0']
    tags.sort(key=lambda st: map(int, st.lstrip('v').split('.')))
    tags = [repo.tags['v'+version] for version in tags]
    return tags

def show_version_diffs(tags):
    tags = list(tags)
    old = tags.pop(0)
    for new in tags:
        diff_index = old.commit.diff(new)
        print '{:7} - {:7}: {} commits'.format(
            old.name, new.name, len(diff_index))
        old = new

# IDEA: use iter_change_type('A') to get all file paths, even if they've been renamed/deleted
# Or: git.diff('v3.0.0..v4.0.0',name_status=True)
# ALSO
# - git.diff('v3.0.0..v3.1.0',dirstat=True)
# TODO: git blame --porcelain v3.0.0..v4.0.0 ip-fou.8

range_1 = 'v3.0.0..v3.1.0'
range_all = 'v3.0.0..v4.0.0'

# TODO: skip "total" at end
def parse_diff_changes(diff_text):
    re_path_num = re.compile(
        r'^\s(\S+) .+? \| \s+ (\d+)',
        re.MULTILINE | re.VERBOSE)
    return re_path_num.finditer(diff_text)

def format_path_changes(all_paths, path_changes):
    def format_diff_value(value):
        return '?.-+*ABCD'[len(value)]

    change_dict = dict(match.groups() for match in path_changes)
    def format_chars():
        for path in all_paths:
            if path not in change_dict:
                yield ' '
            else:
                yield format_diff_value(change_dict[path])
    return ''.join(format_chars())

class Matcher(object):
    def __init__(self, pattern):
        self.match = re.compile(pattern).search

class ManpageMatcher(Matcher):
    def __init__(self):
        super(ManpageMatcher, self).__init__('man/')

all_matcher = str

def get_paths(repo):
    diff_text = repo.git.diff(range_all, stat=True)
    all_paths = (match.group(1) for match in parse_diff_changes(diff_text))
    return sorted(all_paths)

def get_tag_paths(repo, tag):
    ls_tree_text = repo.git.ls_tree(
        tag, full_tree=True, name_status=True, r=True)
    return ls_tree_text.split('\n')

# X probably buggy
def get_latest_datetime(repo, tag):
    latest_opts = {'1': True, 'date': 'short', 'format': '%cd'}
    log_text = repo.git.log(tag, **latest_opts)
    log_date = datetime.datetime.strptime(log_text, '%Y-%m-%d')
    return log_date

def OLD_render_image(repo, matchfunc):
    good_paths = filter(matchfunc, get_paths(repo))
    print good_paths
    tags = get_tags(repo)

    good_index = dict((path, index)
        for index,path in enumerate(good_paths))

# TODO: currently does git-diff, then re-does diff to get each
# "good" path's contribution.  Simplify to just do git-diff once.

    symbols = []
    old = tags.pop(0)
    for y,new in enumerate(tags):
        diff_index = old.commit.diff(new)
        diff_versions = '{}..{}'.format(old.name, new.name)
        old = new
        print '{:7}: {:3}'.format(new.name, len(diff_index))
        diff_paths = set(d.a_path for d in diff_index) & set(good_paths)
        if not diff_paths:
            # version has diffs, but not in the region of interest
            print '- skip:', new
            continue
        diff_text = repo.git.diff(diff_versions, *diff_paths, stat=True)
        diff_path_changes = parse_diff_changes(diff_text)
        for path,diff_count in (
            match.groups() for match in diff_path_changes):
            try:
                x = good_index[path]
            except IndexError:
                print '?', path
                continue
            area = 6 * len(diff_count)
            symbols.append((x, y, area))

    xs, ys, areas = zip(*symbols)
    plt.scatter(xs, ys, s=areas)
    plt.xlabel('file index')
    plt.ylabel('version index')
    plt.savefig('z.png')
    plt.savefig('z.svg')

def serpentine_iter(width):
    y = 0
    while True:
        for x in xrange(width):
            yield x, y
        for x in xrange(width):
            yield width - x - 1, y + 1
        y += 2

def render_image(repo, matchfunc):
    width = 100
    height = 1000
    colors = colorbrewer.diverging.RdBu_11_r.hex_colors

    def format_age(mycommit, mylatest):
        """
        format commit age as single character
        0-9 days / 10-99 days / 100-999 days
        """
        authored_dt = mycommit.authored_datetime.replace(tzinfo=None)
        delta = mylatest - authored_dt
        delta_num = math.log10(delta.days + 1) + 1.0
        return colors[int(delta_num)]

    tag = 'v4.0.0'
    def iter_source():
        latest = get_latest_datetime(repo, tag)
        for path in filter(matchfunc, get_tag_paths(repo, tag)):
            blame = repo.blame(tag, path)
            for commit, regions in blame:
                yield format_age(commit, latest), len(regions)

    im = Image.new('RGB', (width, height))
    im_draw = ImageDraw.Draw(im)

    image_iter = serpentine_iter(width=width)
    for color, size in iter_source():
        points = [image_iter.next() for index in [0]] # X size
        print points, color
        im_draw.point(points, fill=(200, 60, 20))
    im.save('z.png')

def render_text(repo, matchfunc):
    def format_age(mycommit, mylatest):
        """
        format commit age as single character
        * recent (0-9 days); + 10-99 days, - 100-999 days
        """
        authored_dt = mycommit.authored_datetime.replace(tzinfo=None)
        delta = mylatest - authored_dt
        delta_num = len(str(delta.days))
        return '?*+-.'[delta_num]

    tag = 'v4.0.0'
    latest = get_latest_datetime(repo, tag)
    for path in filter(matchfunc, get_tag_paths(repo, tag)):
        blame = repo.blame(tag, path)
        def path_age():
            for commit, regions in blame:
                yield format_age(commit, latest) * len(regions)
        print path, ':', ''.join(path_age())

class Command(BaseCommand):
    help = __doc__

    def add_arguments(self, parser):
        parser.add_argument('--match', choices=('manpage', 'source'))
        parser.add_argument('--style', choices=('text', 'image'),
            default='image')
        parser.add_argument('project_dirs', nargs='+')

    def handle(self, *args, **options):
        render_func = globals()['render_{}'.format(options['style'])]
        matchfunc = {
        # manpage -- in "man" subdir ending in .8 or .in
        'manpage': Matcher(r'man/.*[0-9n]$').match,
        'source': Matcher(r'\.[ch]$').match,
        }.get(options['match']) or all_matcher

        for project_dir in options['project_dirs']:
            repo = Repo(os.path.expanduser(project_dir))
            render_func(repo, matchfunc)
<commit_msg>render version as image, cooler colors = older<commit_after>"""
age.py -- show age of code
"""
# pylint: disable=W0640

# list files in Git branch
# git ls-tree --name-status --full-tree -r v4.0.0

import datetime
import math
import os
import re

import matplotlib.pyplot as plt
from django.core.management.base import BaseCommand
from git import Repo
from palettable import colorbrewer
from PIL import Image, ImageColor, ImageDraw

def get_tags(repo):
    tags = [tag.name.lstrip('v') for tag in repo.tags
        if tag.name.startswith('v3.') or tag.name == 'v4.0.0']
    tags.sort(key=lambda st: map(int, st.lstrip('v').split('.')))
    tags = [repo.tags['v'+version] for version in tags]
    return tags

def show_version_diffs(tags):
    tags = list(tags)
    old = tags.pop(0)
    for new in tags:
        diff_index = old.commit.diff(new)
        print '{:7} - {:7}: {} commits'.format(
            old.name, new.name, len(diff_index))
        old = new

# IDEA: use iter_change_type('A') to get all file paths, even if they've been renamed/deleted
# Or: git.diff('v3.0.0..v4.0.0',name_status=True)
# ALSO
# - git.diff('v3.0.0..v3.1.0',dirstat=True)
# TODO: git blame --porcelain v3.0.0..v4.0.0 ip-fou.8

range_1 = 'v3.0.0..v3.1.0'
range_all = 'v3.0.0..v4.0.0'

# TODO: skip "total" at end
def parse_diff_changes(diff_text):
    re_path_num = re.compile(
        r'^\s(\S+) .+? \| \s+ (\d+)',
        re.MULTILINE | re.VERBOSE)
    return re_path_num.finditer(diff_text)

def format_path_changes(all_paths, path_changes):
    def format_diff_value(value):
        return '?.-+*ABCD'[len(value)]

    change_dict = dict(match.groups() for match in path_changes)
    def format_chars():
        for path in all_paths:
            if path not in change_dict:
                yield ' '
            else:
                yield format_diff_value(change_dict[path])
    return ''.join(format_chars())

class Matcher(object):
    def __init__(self, pattern):
        self.match = re.compile(pattern).search

class ManpageMatcher(Matcher):
    def __init__(self):
        super(ManpageMatcher, self).__init__('man/')

all_matcher = str

def get_paths(repo):
    diff_text = repo.git.diff(range_all, stat=True)
    all_paths = (match.group(1) for match in parse_diff_changes(diff_text))
    return sorted(all_paths)

def get_tag_paths(repo, tag):
    ls_tree_text = repo.git.ls_tree(
        tag, full_tree=True, name_status=True, r=True)
    return ls_tree_text.split('\n')

# X probably buggy
def get_latest_datetime(repo, tag):
    latest_opts = {'1': True, 'date': 'short', 'format': '%cd'}
    log_text = repo.git.log(tag, **latest_opts)
    log_date = datetime.datetime.strptime(log_text, '%Y-%m-%d')
    return log_date

def OLD_render_image(repo, matchfunc):
    good_paths = filter(matchfunc, get_paths(repo))
    print good_paths
    tags = get_tags(repo)

    good_index = dict((path, index)
        for index,path in enumerate(good_paths))

# TODO: currently does git-diff, then re-does diff to get each
# "good" path's contribution.  Simplify to just do git-diff once.

    symbols = []
    old = tags.pop(0)
    for y,new in enumerate(tags):
        diff_index = old.commit.diff(new)
        diff_versions = '{}..{}'.format(old.name, new.name)
        old = new
        print '{:7}: {:3}'.format(new.name, len(diff_index))
        diff_paths = set(d.a_path for d in diff_index) & set(good_paths)
        if not diff_paths:
            # version has diffs, but not in the region of interest
            print '- skip:', new
            continue
        diff_text = repo.git.diff(diff_versions, *diff_paths, stat=True)
        diff_path_changes = parse_diff_changes(diff_text)
        for path,diff_count in (
            match.groups() for match in diff_path_changes):
            try:
                x = good_index[path]
            except IndexError:
                print '?', path
                continue
            area = 6 * len(diff_count)
            symbols.append((x, y, area))

    xs, ys, areas = zip(*symbols)
    plt.scatter(xs, ys, s=areas)
    plt.xlabel('file index')
    plt.ylabel('version index')
    plt.savefig('z.png')
    plt.savefig('z.svg')

def serpentine_iter(width):
    y = 0
    while True:
        for x in xrange(width):
            yield x, y
        for x in xrange(width):
            yield width - x - 1, y + 1
        y += 2

def render_image(repo, matchfunc):
    width = 100
    height = 1000
    COLORMAP = [tuple(color) for color in
        colorbrewer.diverging.RdBu_11_r.colors]

    def format_age(mycommit, mylatest):
        """
        format commit age as single character
        0-9 days / 10-99 days / 100-999 days
        """
        authored_dt = mycommit.authored_datetime.replace(tzinfo=None)
        delta = mylatest - authored_dt
        delta_num = math.log10(delta.days + 1) + 1.0
        return COLORMAP[int(delta_num)]

    tag = 'v4.0.0'
    def iter_source():
        latest = get_latest_datetime(repo, tag)
        for path in filter(matchfunc, get_tag_paths(repo, tag)):
            blame = repo.blame(tag, path)
            for commit, regions in blame:
                yield format_age(commit, latest), len(regions)

    im = Image.new('RGB', (width, height))
    im_draw = ImageDraw.Draw(im)

    image_iter = serpentine_iter(width=width)
    for color, size in iter_source():
        points = [image_iter.next() for _ in xrange(size)]
        im_draw.point(points, fill=color)
    im.save('z.png')

def render_text(repo, matchfunc):
    def format_age(mycommit, mylatest):
        """
        format commit age as single character
        * recent (0-9 days); + 10-99 days, - 100-999 days
        """
        authored_dt = mycommit.authored_datetime.replace(tzinfo=None)
        delta = mylatest - authored_dt
        delta_num = len(str(delta.days))
        return '?*+-.'[delta_num]

    tag = 'v4.0.0'
    latest = get_latest_datetime(repo, tag)
    for path in filter(matchfunc, get_tag_paths(repo, tag)):
        blame = repo.blame(tag, path)
        def path_age():
            for commit, regions in blame:
                yield format_age(commit, latest) * len(regions)
        print path, ':', ''.join(path_age())

class Command(BaseCommand):
    help = __doc__

    def add_arguments(self, parser):
        parser.add_argument('--match', choices=('manpage', 'source'))
        parser.add_argument('--style', choices=('text', 'image'),
            default='image')
        parser.add_argument('project_dirs', nargs='+')

    def handle(self, *args, **options):
        render_func = globals()['render_{}'.format(options['style'])]
        matchfunc = {
        # manpage -- in "man" subdir ending in .8 or .in
        'manpage': Matcher(r'man/.*[0-9n]$').match,
        'source': Matcher(r'\.[ch]$').match,
        }.get(options['match']) or all_matcher

        for project_dir in options['project_dirs']:
            repo = Repo(os.path.expanduser(project_dir))
            render_func(repo, matchfunc)
<|endoftext|>


<commit_before># VOEvent messages.
# John Swinbank, <swinbank@transientskp.org>.

# Python standard library
from datetime import datetime

# XML parsing using lxml
import lxml.etree as ElementTree

from .xml import xml_document

ElementTree.register_namespace("voe", "http://www.ivoa.net/xml/VOEvent/v2.0")

def broker_test_message(ivo):
    """
    Test message which is regularly broadcast to all subscribers.
    """
    root_element = ElementTree.Element("{http://www.ivoa.net/xml/VOEvent/v2.0}VOEvent",
        attrib={
            "ivorn": ivo + "#TestEvent-%s" % datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%S"),
            "role": "test",
            "version": "2.0",
            "{http://www.w3.org/2001/XMLSchema-instance}schemaLocation": "http://www.ivoa.net/xml/VOEvent/v2.0 http://www.ivoa.net/xml/VOEvent/VOEvent-v2.0.xsd"
        }
    )
    who = ElementTree.SubElement(root_element, "Who")
    author_ivorn = ElementTree.SubElement(who, "AuthorIVORN")
    author_ivorn.text = ivo
    date = ElementTree.SubElement(who, "Date")
    date.text = datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%S.%f")
    return xml_document(root_element)
<commit_msg>Include Comet version & URL in test events<commit_after># VOEvent messages.
# John Swinbank, <swinbank@transientskp.org>.

# Python standard library
from datetime import datetime

# XML parsing using lxml
import lxml.etree as ElementTree

from comet import __version__, __url__
from .xml import xml_document

ElementTree.register_namespace("voe", "http://www.ivoa.net/xml/VOEvent/v2.0")

def broker_test_message(ivo):
    """
    Test message which is regularly broadcast to all subscribers.
    """
    root_element = ElementTree.Element("{http://www.ivoa.net/xml/VOEvent/v2.0}VOEvent",
        attrib={
            "ivorn": ivo + "#TestEvent-%s" % datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%S"),
            "role": "test",
            "version": "2.0",
            "{http://www.w3.org/2001/XMLSchema-instance}schemaLocation": "http://www.ivoa.net/xml/VOEvent/v2.0 http://www.ivoa.net/xml/VOEvent/VOEvent-v2.0.xsd"
        }
    )
    who = ElementTree.SubElement(root_element, "Who")
    author_ivorn = ElementTree.SubElement(who, "AuthorIVORN")
    author_ivorn.text = ivo
    date = ElementTree.SubElement(who, "Date")
    date.text = datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%S.%f")
    what = ElementTree.SubElement(root_element, "What")
    description =  ElementTree.SubElement(what, "Description")
    description.text = "Broker test event generated by Comet %s." % (__version__,)
    reference = ElementTree.SubElement(what, "Reference", uri=__url__)
    return xml_document(root_element)
<|endoftext|>


<commit_before>import os
import unittest
import tempfile
import shutil

import numpy
from PIL import Image

from simphony_paraview.snapshot import snapshot
from simphony.cuds.lattice import make_square_lattice
from simphony.cuds.mesh import Mesh, Point
from simphony.cuds.particles import Particles, Particle

class TestSnapShot(unittest.TestCase):

    def setUp(self):
        self.temp_dir = tempfile.mkdtemp()
        self.filename = os.path.join(self.temp_dir, 'test_file.png')
        self.addCleanup(self.cleanup)

    def cleanup(self):
        shutil.rmtree(self.temp_dir)

    def test_lattice_snapshot(self):
        filename = self.filename
        lattice = make_square_lattice(
            'test', 0.2, (10, 10), origin=(0.2, -2.4))
        snapshot(lattice, filename)
        self.assertImageSavedWithContent(filename)

    def test_mesh_snapshot(self):
        filename = self.filename
        points = numpy.array([
            [0, 0, 0], [1, 0, 0], [0, 1, 0], [0, 0, 1],
            [2, 0, 0], [3, 0, 0], [3, 1, 0], [2, 1, 0],
            [2, 0, 1], [3, 0, 1], [3, 1, 1], [2, 1, 1]],
            'f')
        mesh = Mesh('test')
        items = [Point(coordinates=point) for index, point in enumerate(points)]
        mesh.add_points(items)

        snapshot(mesh, filename)
        self.assertImageSavedWithContent(filename)

    def test_particles_snapshot(self):
        filename = self.filename
        coordinates = numpy.array([
            [0, 0, 0], [1, 0, 0], [0, 1, 0], [0, 0, 1],
            [2, 0, 0], [3, 0, 0], [3, 1, 0], [2, 1, 0],
            [2, 0, 1], [3, 0, 1], [3, 1, 1], [2, 1, 1]],
            'f')
        particles = Particles('test')
        items = [Particle(coordinates=point + 3) for index, point in enumerate(coordinates)]
        particles.add_particles(items)

        snapshot(particles, filename)
        self.assertImageSavedWithContent(filename)

    def assertImageSavedWithContent(self, filename):
        """ Load the image and check that there is some content in it.

        """
        image = numpy.array(Image.open(filename))

        self.assertEqual(image.shape[:2], (600, 800))
        if image.shape[2] == 3:
            check = numpy.sum(image == [0, 0, 0], axis=2) == 3
        elif image.shape[2] == 4:
            check = numpy.sum(image == [0, 0, 0, 255], axis=2) == 4
        else:
            self.fail(
                'Pixel size is not 3 or 4, but {0}'.format(image.shape[2]))
        if check.any():
            self.fail('The image has blank spots, something is wrong')
<commit_msg>add a test for invalid cuds<commit_after>import os
import unittest
import tempfile
import shutil

import numpy
from PIL import Image

from simphony_paraview.snapshot import snapshot
from simphony.cuds.lattice import make_square_lattice
from simphony.cuds.mesh import Mesh, Point
from simphony.cuds.particles import Particles, Particle

class TestSnapShot(unittest.TestCase):

    def setUp(self):
        self.temp_dir = tempfile.mkdtemp()
        self.filename = os.path.join(self.temp_dir, 'test_file.png')
        self.addCleanup(self.cleanup)

    def cleanup(self):
        shutil.rmtree(self.temp_dir)

    def test_lattice_snapshot(self):
        filename = self.filename
        lattice = make_square_lattice(
            'test', 0.2, (10, 10), origin=(0.2, -2.4))
        snapshot(lattice, filename)
        self.assertImageSavedWithContent(filename)

    def test_mesh_snapshot(self):
        filename = self.filename
        points = numpy.array([
            [0, 0, 0], [1, 0, 0], [0, 1, 0], [0, 0, 1],
            [2, 0, 0], [3, 0, 0], [3, 1, 0], [2, 1, 0],
            [2, 0, 1], [3, 0, 1], [3, 1, 1], [2, 1, 1]],
            'f')
        mesh = Mesh('test')
        items = [Point(coordinates=point) for index, point in enumerate(points)]
        mesh.add_points(items)

        snapshot(mesh, filename)
        self.assertImageSavedWithContent(filename)

    def test_particles_snapshot(self):
        filename = self.filename
        coordinates = numpy.array([
            [0, 0, 0], [1, 0, 0], [0, 1, 0], [0, 0, 1],
            [2, 0, 0], [3, 0, 0], [3, 1, 0], [2, 1, 0],
            [2, 0, 1], [3, 0, 1], [3, 1, 1], [2, 1, 1]],
            'f')
        particles = Particles('test')
        items = [Particle(coordinates=point + 3) for index, point in enumerate(coordinates)]
        particles.add_particles(items)

        snapshot(particles, filename)
        self.assertImageSavedWithContent(filename)

    def test_snapshot_invalid_cuds(self):
        filename = self.filename
        with self.assertRaises(TypeError):
            snapshot(object(), filename)

    def assertImageSavedWithContent(self, filename):
        """ Load the image and check that there is some content in it.

        """
        image = numpy.array(Image.open(filename))

        self.assertEqual(image.shape[:2], (600, 800))
        if image.shape[2] == 3:
            check = numpy.sum(image == [0, 0, 0], axis=2) == 3
        elif image.shape[2] == 4:
            check = numpy.sum(image == [0, 0, 0, 255], axis=2) == 4
        else:
            self.fail(
                'Pixel size is not 3 or 4, but {0}'.format(image.shape[2]))
        if check.any():
            self.fail('The image has blank spots, something is wrong')
<|endoftext|>