<commit_before>import random
import bcrypt
import time, threading
import json
from time import time
import datetime
from ..models import *
import datetime

def factors(n):

    return set(reduce(list.__add__,
        ([i, n//i] for i in range(1, int(n**0.5) + 1) if n % i == 0)))

def division(startOfRange, endOfRange):

    firstNumber = random.randint(startOfRange, endOfRange)
    factorsFirstNumber = factors(firstNumber)
    secondNumber = random.choice(random.sample(factorsFirstNumber,1))
    result = firstNumber/secondNumber
    resultJson = {'firstNumber' : firstNumber, 'secondNumber' : secondNumber, 'operator': '/','answer' : result}
    return resultJson

def addition(startOfRange, endOfRange):

    firstNumber = random.randint(startOfRange, endOfRange)
    secondNumber = random.randint(startOfRange, endOfRange)
    result = firstNumber + secondNumber
    resultJson = {'firstNumber' : firstNumber, 'secondNumber' : secondNumber, 'operator': '+', 'answer' : result}
    return resultJson

def multiplication(startOfRange, endOfRange):

    firstNumber = random.randint(startOfRange, endOfRange)
    secondNumber = random.randint(startOfRange, endOfRange)
    result = firstNumber * secondNumber
    resultJson = {'firstNumber' : firstNumber, 'secondNumber' : secondNumber, 'operator': '*', 'answer' : result}
    return resultJson

def subtraction(startOfRange, endOfRange):

    firstNumber = random.randint(startOfRange, endOfRange)
    secondNumber = random.randint(startOfRange, firstNumber)
    result = firstNumber - secondNumber
    resultJson = {'firstNumber' : firstNumber, 'secondNumber' : secondNumber, 'operator': '-', 'answer' : result}
    return resultJson

def quesSetForTomorrow(request):
    try:
        apiPass = ApiPassword.objects.get(apiName='quesSetForTomorrow').apiPassword
    except:
        return '404-1'
    if bcrypt.checkpw(str(request['password']),str(apiPass)):
        try:
            today = datetime.datetime.now().date()
            tomorrow = today + datetime.timedelta(days=1)
            epochTime = int(tomorrow.strftime('%s'))
            flag=False
            try:
                quesObjCount=QuesSet_v2.objects.filter(questionDate=tomorrow).count()
                if quesObjCount==0:
                    flag=True
            except Exception as e:
                return '404'
            if flag:
                for i in range(1,1441):
                    questionSetObj = QuesSet_v2()
                    questionSetObj.questionDate = tomorrow
                    quesList = []
                    for j in range(0,120):
                        startOfRange=1
                        endOfRange=10
                        quesList.append(random.choice([addition(startOfRange, endOfRange),
                        multiplication(startOfRange, endOfRange), division(startOfRange, endOfRange),
                        subtraction(startOfRange, endOfRange)]))
                    questionSetObj.questionSet = json.dumps(quesList)
                    questionSetObj.questionTimeStamp = epochTime
                    epochTime = epochTime+30
                    questionSetObj.questionEndStamp = epochTime
                    epochTime = epochTime+30
                    questionSetObj.questionBoutEndStamp = epochTime
                    questionSetObj.save()
                return "202-1"
            else:
                return "202-2"
        except Exception as e:
            return '400'
    else:
        return '409-1'

def quesSetForToday(request):
    try:
        apiPass = ApiPassword.objects.get(apiName= 'quesSetForToday').apiPassword
    except Exception as e:
        print e
        return '404-1'
    if bcrypt.checkpw(str(request['password']), str(apiPass)):
        today = datetime.datetime.now().date()
        epochTime = int(today.strftime('%s'))
        flag=False
        try:
            quesObjCount=QuesSet_v2.objects.filter(questionDate=today).count()
            if quesObjCount==0:
                flag=True
        except Exception as e:
            return "400"
        if flag:

            for i in range(1,1441):
                questionSetObj = QuesSet_v2()
                questionSetObj.questionDate = today
                quesList = []
                for j in range(0,120):
                    startOfRange=1
                    endOfRange=10
                    quesList.append(random.choice([addition(startOfRange, endOfRange),
                    multiplication(startOfRange, endOfRange), division(startOfRange, endOfRange),
                    subtraction(startOfRange, endOfRange)]))
                questionSetObj.questionSet = json.dumps(quesList)
                questionSetObj.questionTimeStamp = epochTime
                epochTime = epochTime+30
                questionSetObj.questionEndStamp = epochTime
                epochTime = epochTime+30
                questionSetObj.questionBoutEndStamp = epochTime
                questionSetObj.save()
            return "202-1"
        else:
            return "202-2"
    else:
        return "409-1"

def deleteQuestion(request):
    try:
        apiPass = ApiPassword.objects.get(apiName = 'deleteQuestion').apiPassword
    except Exception as e :
        return '404-1'
    if bcrypt.checkpw(str(request['password']), str(apiPass)):
        try:
            if request['day'] == 'tomorrow':
                quesObj = QuesSet_v2.objects.filter(questionDate=(datetime.datetime.now().date()+datetime.timedelta(days=1)))
            elif request['day'] == 'yesterday':
                quesObj = QuesSet_v2.objects.filter(questionDate=(datetime.datetime.now().date()-datetime.timedelta(days=1)))
            elif request['day'] == 'today':
                quesObj = QuesSet_v2.objects.filter(questionDate=datetime.datetime.now().date())
            quesObj.delete()
            return "202-1"

        except Exception as e:
            print e
            return "400-1"
    else:
        return "409-1"
<commit_msg>changed range from all the operations<commit_after>import random
import bcrypt
import time, threading
import json
from time import time
import datetime
from ..models import *
import datetime

def factors(n):

    return set(reduce(list.__add__,
        ([i, n//i] for i in range(1, int(n**0.5) + 1) if n % i == 0)))

def division(startOfRange, endOfRange):

    firstNumber = random.randint(startOfRange, endOfRange)
    factorsFirstNumber = factors(firstNumber)
    secondNumber = random.choice(random.sample(factorsFirstNumber,1))
    result = firstNumber/secondNumber
    resultJson = {'firstNumber' : firstNumber, 'secondNumber' : secondNumber, 'operator': '/','answer' : result}
    return resultJson

def addition(startOfRange, endOfRange):

    firstNumber = random.randint(startOfRange, endOfRange)
    secondNumber = random.randint(startOfRange, endOfRange)
    result = firstNumber + secondNumber
    resultJson = {'firstNumber' : firstNumber, 'secondNumber' : secondNumber, 'operator': '+', 'answer' : result}
    return resultJson

def multiplication(startOfRange, endOfRange):

    firstNumber = random.randint(startOfRange, endOfRange)
    secondNumber = random.randint(startOfRange, endOfRange)
    result = firstNumber * secondNumber
    resultJson = {'firstNumber' : firstNumber, 'secondNumber' : secondNumber, 'operator': 'X', 'answer' : result}
    return resultJson

def subtraction(startOfRange, endOfRange):

    firstNumber = random.randint(startOfRange, endOfRange)
    secondNumber = random.randint(startOfRange, firstNumber)
    result = firstNumber - secondNumber
    resultJson = {'firstNumber' : firstNumber, 'secondNumber' : secondNumber, 'operator': '-', 'answer' : result}
    return resultJson

def quesSetForTomorrow(request):
    try:
        apiPass = ApiPassword.objects.get(apiName='quesSetForTomorrow').apiPassword
    except:
        return '404-1'
    if bcrypt.checkpw(str(request['password']),str(apiPass)):
        try:
            today = datetime.datetime.now().date()
            tomorrow = today + datetime.timedelta(days=1)
            epochTime = int(tomorrow.strftime('%s'))
            flag=False
            try:
                quesObjCount=QuesSet_v2.objects.filter(questionDate=tomorrow).count()
                if quesObjCount==0:
                    flag=True
            except Exception as e:
                return '404'
            if flag:
                for i in range(1,1441):
                    questionSetObj = QuesSet_v2()
                    questionSetObj.questionDate = tomorrow
                    quesList = []
                    for j in range(0,120):

                        quesList.append(random.choice([addition(1, 100),
                        multiplication(1, 15), division(1, 100),
                        subtraction(1, 100)]))
                    questionSetObj.questionSet = json.dumps(quesList)
                    questionSetObj.questionTimeStamp = epochTime
                    epochTime = epochTime+30
                    questionSetObj.questionEndStamp = epochTime
                    epochTime = epochTime+30
                    questionSetObj.questionBoutEndStamp = epochTime
                    questionSetObj.save()
                return "202-1"
            else:
                return "202-2"
        except Exception as e:
            return '400'
    else:
        return '409-1'

def quesSetForToday(request):
    try:
        apiPass = ApiPassword.objects.get(apiName= 'quesSetForToday').apiPassword
    except Exception as e:
        print e
        return '404-1'
    if bcrypt.checkpw(str(request['password']), str(apiPass)):
        today = datetime.datetime.now().date()
        epochTime = int(today.strftime('%s'))
        flag=False
        try:
            quesObjCount=QuesSet_v2.objects.filter(questionDate=today).count()
            if quesObjCount==0:
                flag=True
        except Exception as e:
            return "400"
        if flag:

            for i in range(1,1441):
                questionSetObj = QuesSet_v2()
                questionSetObj.questionDate = today
                quesList = []
                for j in range(0,120):
                    startOfRange=1
                    endOfRange=10
                    quesList.append(random.choice([addition(startOfRange, endOfRange),
                    multiplication(startOfRange, endOfRange), division(startOfRange, endOfRange),
                    subtraction(startOfRange, endOfRange)]))
                questionSetObj.questionSet = json.dumps(quesList)
                questionSetObj.questionTimeStamp = epochTime
                epochTime = epochTime+30
                questionSetObj.questionEndStamp = epochTime
                epochTime = epochTime+30
                questionSetObj.questionBoutEndStamp = epochTime
                questionSetObj.save()
            return "202-1"
        else:
            return "202-2"
    else:
        return "409-1"

def deleteQuestion(request):
    try:
        apiPass = ApiPassword.objects.get(apiName = 'deleteQuestion').apiPassword
    except Exception as e :
        return '404-1'
    if bcrypt.checkpw(str(request['password']), str(apiPass)):
        try:
            if request['day'] == 'tomorrow':
                quesObj = QuesSet_v2.objects.filter(questionDate=(datetime.datetime.now().date()+datetime.timedelta(days=1)))
            elif request['day'] == 'yesterday':
                quesObj = QuesSet_v2.objects.filter(questionDate=(datetime.datetime.now().date()-datetime.timedelta(days=1)))
            elif request['day'] == 'today':
                quesObj = QuesSet_v2.objects.filter(questionDate=datetime.datetime.now().date())
            quesObj.delete()
            return "202-1"

        except Exception as e:
            print e
            return "400-1"
    else:
        return "409-1"
<|endoftext|>


<commit_before>#!/usr/bin/python

import pygtk
pygtk.require('2.0')
import gtk

import fontTools.ttx

class FontTreeStoreBuilder:

	def __init__(self, treestore):
		self.ts = treestore

	def add_object(self, parent, key, value):
		# Make sure item is decompiled
		try:
			getattr(value, "asdf")
		except AttributeError:
			pass
		if value.__class__ == fontTools.ttLib.getTableModule('glyf').Glyph:
			# Glyph type needs explicit expanding to be useful
			value.expand(self.font['glyf'])

		item = self.ts.append(parent, [key, '%s' % value.__class__.__name__])
		for k,v in sorted(value.__dict__.items()):
			self.add_thing (item, k, v)

	def add_dict(self, parent, key, value):
		item = self.ts.append(parent, [key, '%s of %d items' %
						    (value.__class__.__name__, len(value))])
		for k,v in sorted(value.items()):
			self.add_thing (item, k, v)

	def add_list(self, parent, key, value):
		item = self.ts.append(parent, [key, '%s of %d items' %
						    (value.__class__.__name__, len(value))])
		for k,v in enumerate(value):
			self.add_thing (item, k, v)

	def add_thing(self, parent, key, value):
		if value == self.font:
			return
		if key == 'reader':
			return
		if hasattr(value, '__dict__'):
			self.add_object(parent, key, value)
			return
		if hasattr(value, 'items'):
			self.add_dict(parent, key, value)
			return
		if not isinstance(value, basestring):
			# Sequences
			try:
				len(value)
				iter(value)
				self.add_list(parent, key, value)
				return
			except TypeError:
				pass
			except AttributeError:
				pass
			except KeyError:
				pass

		# Everything else
		item = self.ts.append(parent, [key, value])

	def add_font(self, font):
		self.font = font
		for tag in font.keys():
			self.add_thing (None, tag, font[tag])
		self.font = None

class PyFontView:

    def delete_event(self, widget, event, data=None):
	gtk.main_quit()
	return False

    def __init__(self):

	self.window = gtk.Window(gtk.WINDOW_TOPLEVEL)
	self.window.set_title("PyFontView")
	self.window.connect("delete_event", self.delete_event)
	self.window.set_size_request(400, 600)

	self.scrolled_window = gtk.ScrolledWindow()
	self.window.add(self.scrolled_window)

	self.treestore = gtk.TreeStore(str, str)
	self.treeview = gtk.TreeView(self.treestore)
	#self.treeview.set_reorderable(True)

	font = fontTools.ttx.TTFont("abc.woff")
	#font = fontTools.ttx.TTFont("IranNastaliq2.ttf")
	builder = FontTreeStoreBuilder(self.treestore)
	builder.add_font (font)

	for i in range(2):

		col = gtk.TreeViewColumn('Column %d' % i)

		self.treeview.append_column(col)

		cell = gtk.CellRendererText()
		col.pack_start(cell, True)

		col.add_attribute(cell, 'text', i)

		self.treeview.set_search_column(i)
		col.set_sort_column_id(i)

	self.scrolled_window.add(self.treeview)
	self.window.show_all()

def main():
    gtk.main()

if __name__ == "__main__":
    viewer = PyFontView()
    main()
<commit_msg>Improve handling of sequence types<commit_after>#!/usr/bin/python

import pygtk
pygtk.require('2.0')
import gtk

import fontTools.ttx

class FontTreeStoreBuilder:

	def __init__(self, treestore):
		self.ts = treestore

	def add_object(self, parent, key, value):
		# Make sure item is decompiled
		try:
			getattr(value, "asdf")
		except AttributeError:
			pass
		if value.__class__ == fontTools.ttLib.getTableModule('glyf').Glyph:
			# Glyph type needs explicit expanding to be useful
			value.expand(self.font['glyf'])

		item = self.ts.append(parent, [key, '%s' % value.__class__.__name__])
		for k,v in sorted(value.__dict__.items()):
			self.add_thing (item, k, v)

	def add_dict(self, parent, key, value):
		item = self.ts.append(parent, [key, '%s of %d items' %
						    (value.__class__.__name__, len(value))])
		for k,v in sorted(value.items()):
			self.add_thing (item, k, v)

	def add_list(self, parent, key, value):
		item = self.ts.append(parent, [key, '%s of %d items' %
						    (value.__class__.__name__, len(value))])
		for k,v in enumerate(value):
			self.add_thing (item, k, v)

	def add_thing(self, parent, key, value):
		if value == self.font:
			return
		if key == 'reader':
			return
		if not isinstance(value, basestring):
			# Sequences
			try:
				len(value)
				iter(value)
				# It's hard to differentiate list-type sequences
				# from dict-type ones.  Try fetching item 0.
				value[0]
				self.add_list(parent, key, value)
				return
			except TypeError:
				pass
			except AttributeError:
				pass
			except KeyError:
				pass
			except IndexError:
				pass
		if hasattr(value, '__dict__'):
			self.add_object(parent, key, value)
			return
		if hasattr(value, 'items'):
			self.add_dict(parent, key, value)
			return

		# Everything else
		item = self.ts.append(parent, [key, value])

	def add_font(self, font):
		self.font = font
		for tag in font.keys():
			self.add_thing (None, tag, font[tag])
		self.font = None

class PyFontView:

    def delete_event(self, widget, event, data=None):
	gtk.main_quit()
	return False

    def __init__(self):

	self.window = gtk.Window(gtk.WINDOW_TOPLEVEL)
	self.window.set_title("PyFontView")
	self.window.connect("delete_event", self.delete_event)
	self.window.set_size_request(400, 600)

	self.scrolled_window = gtk.ScrolledWindow()
	self.window.add(self.scrolled_window)

	self.treestore = gtk.TreeStore(str, str)
	self.treeview = gtk.TreeView(self.treestore)
	#self.treeview.set_reorderable(True)

	font = fontTools.ttx.TTFont("abc.woff")
	#font = fontTools.ttx.TTFont("IranNastaliq2.ttf")
	builder = FontTreeStoreBuilder(self.treestore)
	builder.add_font (font)

	for i in range(2):

		col = gtk.TreeViewColumn('Column %d' % i)

		self.treeview.append_column(col)

		cell = gtk.CellRendererText()
		col.pack_start(cell, True)

		col.add_attribute(cell, 'text', i)

		self.treeview.set_search_column(i)
		col.set_sort_column_id(i)

	self.scrolled_window.add(self.treeview)
	self.window.show_all()

def main():
    gtk.main()

if __name__ == "__main__":
    viewer = PyFontView()
    main()
<|endoftext|>


<commit_before>'''
Write a program that prompts for a file name, then opens that file and reads through the file,
and print the contents of the file in upper case.
'''
fileName = raw_input('Enter file name :')
openFile = open(fileName)
readFile = openFile.read().upper().strip()
print readFile
<commit_msg>Extra space added<commit_after>'''
Write a program that prompts for a file name, then opens that file and reads through the file,
and print the contents of the file in upper case.
'''
fileName = raw_input('Enter file name : ')
openFile = open(fileName)
readFile = openFile.read().upper().strip()
print readFile
<|endoftext|>


<commit_before>"""
   pyhis.util
   ~~~~~~~~~~

   Collection of useful functions for common use cases
"""
import os

import appdirs
import pandas

import pyhis

#http://midgewater.twdb.state.tx.us/tpwd/soap/wateroneflow.wsdl
#http://midgewater.twdb.state.tx.us/tceq/soap/wateroneflow.wsdl
#http://midgewater.twdb.state.tx.us/cbi/soap/wateroneflow.wsdl

def get_parameter_within_polygon(wsdl_list, param_code_list, verts, merge='BySource'):
    """given a list of service wsdl urls and list of service specific
    param codes and a list of polygon vertices returns pandas.DataFrame of data.
    optional flags
        merge = False : Each site timeseries is a column in dataframe
        merge = 'BySource' : each column in dataframe corresponds to a service
        merge = True : single pandas.timeseries

    Note: merging averages across values that occur at the same timestamp.
    """

    dataframe = pandas.DataFrame()
    for wsdl_url, param_code in zip(wsdl_list, param_code_list):
        source = pyhis.Source(wsdl_url)
        sites = source.get_sites_within_polygon(verts)
        ts_dict = {}
        for site in sites:
            if param_code in sites.timeseries:
                ts_dict[''.join((ts.site.network,':',ts.site.code))] = ts.data
                #ts_list.append(site.timeseries[param_code])
        pd = pandas.Dataset(ts_dict)
        if merge.lower()=='BySource':
            pd = pandas.DataSet({site.network:pd.mean(axis=1)})
        dataframe.append(pd)

    if merge is True:
        dataframe = dataframe.mean(axis=1)

    return dataframe

def _get_pyhis_dir():
    return_dir = appdirs.user_data_dir('pyhis', 'pyhis')
    _mkdir_if_doesnt_exist(return_dir)
    return return_dir

def _mkdir_if_doesnt_exist(dir_path):
    if not os.path.exists(dir_path):
        os.mkdir(dir_path)
<commit_msg>add helper util methods for get/creating tables and groups<commit_after>"""
   pyhis.util
   ~~~~~~~~~~

   Collection of useful functions for common use cases
"""
import os

import appdirs
import pandas

import pyhis

#http://midgewater.twdb.state.tx.us/tpwd/soap/wateroneflow.wsdl
#http://midgewater.twdb.state.tx.us/tceq/soap/wateroneflow.wsdl
#http://midgewater.twdb.state.tx.us/cbi/soap/wateroneflow.wsdl

def get_parameter_within_polygon(wsdl_list, param_code_list, verts, merge='BySource'):
    """given a list of service wsdl urls and list of service specific
    param codes and a list of polygon vertices returns pandas.DataFrame of data.
    optional flags
        merge = False : Each site timeseries is a column in dataframe
        merge = 'BySource' : each column in dataframe corresponds to a service
        merge = True : single pandas.timeseries

    Note: merging averages across values that occur at the same timestamp.
    """

    dataframe = pandas.DataFrame()
    for wsdl_url, param_code in zip(wsdl_list, param_code_list):
        source = pyhis.Source(wsdl_url)
        sites = source.get_sites_within_polygon(verts)
        ts_dict = {}
        for site in sites:
            if param_code in sites.timeseries:
                ts_dict[''.join((ts.site.network,':',ts.site.code))] = ts.data
                #ts_list.append(site.timeseries[param_code])
        pd = pandas.Dataset(ts_dict)
        if merge.lower()=='BySource':
            pd = pandas.DataSet({site.network:pd.mean(axis=1)})
        dataframe.append(pd)

    if merge is True:
        dataframe = dataframe.mean(axis=1)

    return dataframe

def get_or_create_group(h5file, path, title):
    return _get_or_create_node('createGroup', h5file, path, title)

def get_or_create_table(h5file, path, table_definition, title):
    return _get_or_create_node('createTable', h5file, path, table_definition,
            title)

def _get_or_create_node(method_name, h5file, path, *args, **kwargs):
    try:
        node = h5file.getNode(path)
    except NoSuchNodeError:
        where, name = path.rsplit('/', 1)
        with warnings.catch_warnings():
            warnings.simplefilter("ignore")
            create_method = getattr(h5file, method_name)
            node = create_method(*args, **kwargs)
    return node

def _get_pyhis_dir():
    return_dir = appdirs.user_data_dir('pyhis', 'pyhis')
    _mkdir_if_doesnt_exist(return_dir)
    return return_dir

def _mkdir_if_doesnt_exist(dir_path):
    if not os.path.exists(dir_path):
        os.mkdir(dir_path)
<|endoftext|>


<commit_before>from qdb import app
from datetime import datetime
from flask import (url_for, render_template, jsonify,
	redirect, abort, flash, request, session)
from qdb.models import Quote
from qdb.database import db_session
from qdb.utils import Paginator
from sqlalchemy.sql.expression import func

@app.route('/')
def home():
	query = db_session.query(Quote) \
		.filter(Quote.approved == True)

	order = request.args.get('order', 'desc').lower()
	if order == 'asc':
		query = query.order_by(Quote.submitted_at.asc()) \
			.order_by(Quote.id.asc())
	else:
		query = query.order_by(Quote.submitted_at.desc()) \
			.order_by(Quote.id.desc())

	search = request.args.get('s')
	if search:
		query = query.filter(Quote.body.ilike('%'+search+'%'))

	page = int(request.args.get('p', 1))
	if page > 0:
		paginator = Paginator(query, page, request.url)
		quotes = paginator.items

		if request.headers.get('Accept') == 'application/json':
			return jsonify(quotes=paginator)
	else:
		paginator = None
		quotes = query.all()

		if request.headers.get('Accept') == 'application/json':
			return jsonify(quotes=quotes)

	return render_template('list.html.jinja', quotes=quotes, paginator=paginator)

@app.route('/random')
def random():
	query = db_session.query(Quote) \
		.filter(Quote.approved == True) \
		.order_by(func.random()) \
		.limit(10)

	search = request.args.get('s')
	if search:
		query = query.filter(Quote.body.ilike('%'+search+'%'))

	quotes = query.all()

	if request.headers.get('Accept') == 'application/json':
		return jsonify(quotes=quotes)

	return render_template('list.html.jinja', quotes=quotes)

@app.route('/submit', methods=['GET', 'POST'])
def submit():
	if request.method == 'GET' or request.method == 'HEAD':
		return render_template('form.html.jinja',
			form_action=url_for('submit'),
			body='',
		)

	body = Quote.prepare(request.form['body'])
	quote = Quote(body, datetime.now())
	db_session.add(quote)
	db_session.commit()

	flash('Quote added!')
	return redirect(url_for('home'))

@app.route('/pending')
def pending():
	if not session.get('logged_in'):
		flash('You must be logged in to do that!')
		return redirect(url_for('home'))

	query = db_session.query(Quote) \
		.filter(Quote.approved == False) \
		.order_by(Quote.submitted_at.desc())

	quotes = query.all()

	if request.headers.get('Accept') == 'application/json':
		return jsonify(quotes=quotes)

	return render_template('list.html.jinja', quotes=quotes)

@app.route('/login')
def login():
	if request.args.get('password') == app.config.get('PASSWORD'):
		session['logged_in'] = True
		flash('You are now logged in!')
	else:
		flash('Incorrect password!')

	return redirect(url_for('home'))

@app.route('/<int:quote_id>', methods=['GET', 'DELETE', 'PATCH'])
def show(quote_id):
	quote = db_session.query(Quote) \
		.filter(Quote.id == quote_id) \
		.first()

	if not quote:
		return abort(404)

	if request.method == 'GET' or request.method == 'HEAD':
		if request.headers.get('Accept') == 'application/json':
			return jsonify(quote=quote)
		return render_template('show.html.jinja', quote=quote)

	if not session.get('logged_in'):
		return abort(401)

	if request.method == 'DELETE':
		db_session.delete(quote)
		db_session.commit()
		return 'OK'

	body = Quote.prepare(request.form['body'])
	quote.body = body
	db_session.add(quote)
	db_session.commit()
	return 'OK'

@app.route('/<int:quote_id>/approve', methods=['POST'])
def approve(quote_id):
	if not session.get('logged_in'):
		return abort(401)

	quote = db_session.query(Quote) \
		.filter(Quote.id == quote_id) \
		.first()

	if not quote:
		return abort(404)

	quote.approved = True
	db_session.add(quote)
	db_session.commit()
	return 'OK'
<commit_msg>cannot call filter after limit... facepalm<commit_after>from qdb import app
from datetime import datetime
from flask import (url_for, render_template, jsonify,
	redirect, abort, flash, request, session)
from qdb.models import Quote
from qdb.database import db_session
from qdb.utils import Paginator
from sqlalchemy.sql.expression import func

@app.route('/')
def home():
	query = db_session.query(Quote) \
		.filter(Quote.approved == True)

	order = request.args.get('order', 'desc').lower()
	if order == 'asc':
		query = query.order_by(Quote.submitted_at.asc()) \
			.order_by(Quote.id.asc())
	else:
		query = query.order_by(Quote.submitted_at.desc()) \
			.order_by(Quote.id.desc())

	search = request.args.get('s')
	if search:
		query = query.filter(Quote.body.ilike('%'+search+'%'))

	page = int(request.args.get('p', 1))
	if page > 0:
		paginator = Paginator(query, page, request.url)
		quotes = paginator.items

		if request.headers.get('Accept') == 'application/json':
			return jsonify(quotes=paginator)
	else:
		paginator = None
		quotes = query.all()

		if request.headers.get('Accept') == 'application/json':
			return jsonify(quotes=quotes)

	return render_template('list.html.jinja', quotes=quotes, paginator=paginator)

@app.route('/random')
def random():
	query = db_session.query(Quote)

	search = request.args.get('s')
	if search:
		query = query.filter(Quote.body.ilike('%'+search+'%'))

	query = query.filter(Quote.approved == True) \
		.order_by(func.random()) \
		.limit(10)

	quotes = query.all()

	if request.headers.get('Accept') == 'application/json':
		return jsonify(quotes=quotes)

	return render_template('list.html.jinja', quotes=quotes)

@app.route('/submit', methods=['GET', 'POST'])
def submit():
	if request.method == 'GET' or request.method == 'HEAD':
		return render_template('form.html.jinja',
			form_action=url_for('submit'),
			body='',
		)

	body = Quote.prepare(request.form['body'])
	quote = Quote(body, datetime.now())
	db_session.add(quote)
	db_session.commit()

	flash('Quote added!')
	return redirect(url_for('home'))

@app.route('/pending')
def pending():
	if not session.get('logged_in'):
		flash('You must be logged in to do that!')
		return redirect(url_for('home'))

	query = db_session.query(Quote) \
		.filter(Quote.approved == False) \
		.order_by(Quote.submitted_at.desc())

	quotes = query.all()

	if request.headers.get('Accept') == 'application/json':
		return jsonify(quotes=quotes)

	return render_template('list.html.jinja', quotes=quotes)

@app.route('/login')
def login():
	if request.args.get('password') == app.config.get('PASSWORD'):
		session['logged_in'] = True
		flash('You are now logged in!')
	else:
		flash('Incorrect password!')

	return redirect(url_for('home'))

@app.route('/<int:quote_id>', methods=['GET', 'DELETE', 'PATCH'])
def show(quote_id):
	quote = db_session.query(Quote) \
		.filter(Quote.id == quote_id) \
		.first()

	if not quote:
		return abort(404)

	if request.method == 'GET' or request.method == 'HEAD':
		if request.headers.get('Accept') == 'application/json':
			return jsonify(quote=quote)
		return render_template('show.html.jinja', quote=quote)

	if not session.get('logged_in'):
		return abort(401)

	if request.method == 'DELETE':
		db_session.delete(quote)
		db_session.commit()
		return 'OK'

	body = Quote.prepare(request.form['body'])
	quote.body = body
	db_session.add(quote)
	db_session.commit()
	return 'OK'

@app.route('/<int:quote_id>/approve', methods=['POST'])
def approve(quote_id):
	if not session.get('logged_in'):
		return abort(401)

	quote = db_session.query(Quote) \
		.filter(Quote.id == quote_id) \
		.first()

	if not quote:
		return abort(404)

	quote.approved = True
	db_session.add(quote)
	db_session.commit()
	return 'OK'
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""Utility functions for pyIEM package

This module contains utility functions used by various parts of the codebase.
"""
import psycopg2
import netrc
import time
import random
import logging
import datetime
import re
from socket import error as socket_error
from pyiem.ftpsession import FTPSession

SEQNUM = re.compile(r"\001?[0-9]{3}\s?")

def noaaport_text(text):
    """Make whatever text look like it is NOAAPort Pristine

    Args:
      text (string): the inbound text
    Returns:
      text that looks noaaportish
    """
    # Convert to LFLFCR
    text = text.replace("\n", "\r\r\n").replace("\r\r\r\r", "\r\r")
    lines = text.split("\r\r\n")
    # lime 0 should be start of product sequence
    if lines[0] != "\001":
        lines.insert(0, "\001")
    # line 1 should be the LDM sequence number 4 chars
    if not SEQNUM.match(lines[1]):
        if len(lines[1]) > 5:
            lines.insert(1, "000 ")
    # last line should be the control-c, by itself
    if lines[-1] != "\003":
        lines.append("\003")
    # Second line should not be blank
    if lines[1].strip() == 0:
        lines = [lines[0], ] + lines[2:]

    return "\r\r\n".join(lines)

def get_autoplot_context(fdict, cfg):
    """Get the variables out of a dict of strings

    This helper for IEM autoplot gets values out of a dictionary of strings,
    as provided by CGI.  It does some magic to get types right, defaults right
    and so on.  The typical way this is called

        ctx = iemutils.get_context(fdict, get_description())

    Args:
      fdict (dictionary): what was likely provided by `cgi.FieldStorage()`
      cfg (dictionary): autoplot value of get_description
    Returns:
      dictionary of variable names and values, with proper types!
    """
    ctx = {}
    for opt in cfg.get('arguments', []):
        name = opt.get('name')
        default = opt.get('default')
        typ = opt.get('type')
        minval = opt.get('min')
        maxval = opt.get('max')
        optional = opt.get('optional', False)
        value = fdict.get(name)
        if optional and value is None and typ not in ['vtec_ps']:
            continue
        if typ in ['station', 'zstation', 'sid', 'networkselect']:
            # A bit of hackery here if we have a name ending in a number
            netname = "network%s" % (name[-1] if name[-1] != 'n' else '',)
            ctx[netname] = fdict.get(netname)
            # The network variable tags along and within a non-PHP context,
            # this variable is unset, so we do some more hackery here
            if ctx[netname] is None:
                ctx[netname] = opt.get('network')
        elif typ in ['int', 'month', 'zhour', 'hour', 'day', 'year']:
            if value is not None:
                value = int(value)
            if default is not None:
                default = int(default)
        elif typ == 'float':
            if value is not None:
                value = float(value)
            if default is not None:
                default = float(default)
        elif typ == 'select':
            options = opt.get('options', dict())
            if value not in options:
                value = default
        elif typ == 'datetime':
            # tricky here, php has YYYY/mm/dd and CGI has YYYY-mm-dd
            if default is not None:
                default = datetime.datetime.strptime(default,
                                                     '%Y/%m/%d %H%M')
            if minval is not None:
                minval = datetime.datetime.strptime(minval,
                                                    '%Y/%m/%d %H%M')
            if maxval is not None:
                maxval = datetime.datetime.strptime(maxval,
                                                    '%Y/%m/%d %H%M')
            if value is not None:
                value = datetime.datetime.strptime(value, '%Y-%m-%d %H%M')
        elif typ == 'date':
            # tricky here, php has YYYY/mm/dd and CGI has YYYY-mm-dd
            if default is not None:
                default = datetime.datetime.strptime(default,
                                                     '%Y/%m/%d').date()
            if minval is not None:
                minval = datetime.datetime.strptime(minval,
                                                    '%Y/%m/%d').date()
            if maxval is not None:
                maxval = datetime.datetime.strptime(maxval,
                                                    '%Y/%m/%d').date()
            if value is not None:
                value = datetime.datetime.strptime(value, '%Y-%m-%d').date()
        elif typ == 'vtec_ps':
            # VTEC phenomena and significance
            for label in ['phenomena', 'significance']:
                label = label + name
                ctx[label] = fdict.get(label)
            continue
        # validation
        if minval is not None and value is not None and value < minval:
            value = default
        if maxval is not None and value is not None and value > maxval:
            value = default
        ctx[name] = value if value is not None else default
    return ctx

def exponential_backoff(func, *args, **kwargs):
    """ Exponentially backoff some function until it stops erroring"""
    msgs = []
    for i in range(5):
        try:
            return func(*args, **kwargs)
        except socket_error as serr:
            msgs.append("%s/5 %s traceback: %s" % (i+1, func.__name__, serr))
            time.sleep((2 ** i) + (random.randint(0, 1000) / 1000))
        except Exception as exp:
            msgs.append("%s/5 %s traceback: %s" % (i+1, func.__name__, exp))
            time.sleep((2 ** i) + (random.randint(0, 1000) / 1000))
    logging.error("%s failure" % (func.__name__,))
    logging.error("\n".join(msgs))
    return None

def send2box(filenames, remote_path, remotenames=None,
             ftpserver='ftp.box.com', tmpdir='/tmp', fs=None):
    """Send one or more files to CyBox

    Box has a filesize limit of 15 GB, so if we find any files larger than
    that, we shall split them into chunks prior to uploading.

    Args:
      filenames (str or list): filenames to upload
      remote_path (str): location to place the filenames
      remotenames (str or list): filenames to use on the remote FTP server
        should match size and type of filenames
      ftpserver (str): FTP server to connect to...
      tmpdir (str, optional): Temperary folder to if an individual file is over
        15 GB in size

    Returns:
      FTPSession
      list of success `True` or failures `False` matching filenames
    """
    credentials = netrc.netrc().hosts[ftpserver]
    if fs is None:
        fs = FTPSession(ftpserver, credentials[0], credentials[2],
                        tmpdir=tmpdir)
    if isinstance(filenames, str):
        filenames = [filenames, ]
    if remotenames is None:
        remotenames = filenames
    if isinstance(remotenames, str):
        remotenames = [remotenames, ]
    res = fs.put_files(remote_path, filenames, remotenames)
    return fs, res

def get_properties():
    """Fetch the properties set

    Returns:
      dict: a dictionary of property names and values (both str)
    """
    pgconn = psycopg2.connect(database='mesosite', host='iemdb', user='nobody')
    cursor = pgconn.cursor()
    cursor.execute("""SELECT propname, propvalue from properties""")
    res = {}
    for row in cursor:
        res[row[0]] = row[1]
    return res

def drct2text(drct):
    """Convert an degree value to text representation of direction.

    Args:
      drct (int or float): Value in degrees to convert to text

    Returns:
      str: String representation of the direction, could be `None`

    """
    if drct is None:
        return None
    # Convert the value into a float
    drct = float(drct)
    if drct > 360:
        return None
    text = None
    if drct >= 350 or drct < 13:
        text = "N"
    elif drct >= 13 and drct < 35:
        text = "NNE"
    elif drct >= 35 and drct < 57:
        text = "NE"
    elif drct >= 57 and drct < 80:
        text = "ENE"
    elif drct >= 80 and drct < 102:
        text = "E"
    elif drct >= 102 and drct < 127:
        text = "ESE"
    elif drct >= 127 and drct < 143:
        text = "SE"
    elif drct >= 143 and drct < 166:
        text = "SSE"
    elif drct >= 166 and drct < 190:
        text = "S"
    elif drct >= 190 and drct < 215:
        text = "SSW"
    elif drct >= 215 and drct < 237:
        text = "SW"
    elif drct >= 237 and drct < 260:
        text = "WSW"
    elif drct >= 260 and drct < 281:
        text = "W"
    elif drct >= 281 and drct < 304:
        text = "WNW"
    elif drct >= 304 and drct < 324:
        text = "NW"
    elif drct >= 324 and drct < 350:
        text = "NNW"
    return text

if __name__ == '__main__':
    logger = logging.getLogger()
    logger.setLevel(logging.DEBUG)
    send2box(['util.py', 'plot.py'], '/bah1/bah2/', remotenames=['util2.py',
                                                                 'plot.py'])
    # mirror2box("/tmp/mytest", "mytest")
<commit_msg>make autoplot datetime parsing a bit more forgiving<commit_after># -*- coding: utf-8 -*-
"""Utility functions for pyIEM package

This module contains utility functions used by various parts of the codebase.
"""
import psycopg2
import netrc
import time
import random
import logging
import datetime
import re
from socket import error as socket_error
from pyiem.ftpsession import FTPSession

SEQNUM = re.compile(r"\001?[0-9]{3}\s?")

def noaaport_text(text):
    """Make whatever text look like it is NOAAPort Pristine

    Args:
      text (string): the inbound text
    Returns:
      text that looks noaaportish
    """
    # Convert to LFLFCR
    text = text.replace("\n", "\r\r\n").replace("\r\r\r\r", "\r\r")
    lines = text.split("\r\r\n")
    # lime 0 should be start of product sequence
    if lines[0] != "\001":
        lines.insert(0, "\001")
    # line 1 should be the LDM sequence number 4 chars
    if not SEQNUM.match(lines[1]):
        if len(lines[1]) > 5:
            lines.insert(1, "000 ")
    # last line should be the control-c, by itself
    if lines[-1] != "\003":
        lines.append("\003")
    # Second line should not be blank
    if lines[1].strip() == 0:
        lines = [lines[0], ] + lines[2:]

    return "\r\r\n".join(lines)

def get_autoplot_context(fdict, cfg):
    """Get the variables out of a dict of strings

    This helper for IEM autoplot gets values out of a dictionary of strings,
    as provided by CGI.  It does some magic to get types right, defaults right
    and so on.  The typical way this is called

        ctx = iemutils.get_context(fdict, get_description())

    Args:
      fdict (dictionary): what was likely provided by `cgi.FieldStorage()`
      cfg (dictionary): autoplot value of get_description
    Returns:
      dictionary of variable names and values, with proper types!
    """
    ctx = {}
    for opt in cfg.get('arguments', []):
        name = opt.get('name')
        default = opt.get('default')
        typ = opt.get('type')
        minval = opt.get('min')
        maxval = opt.get('max')
        optional = opt.get('optional', False)
        value = fdict.get(name)
        if optional and value is None and typ not in ['vtec_ps']:
            continue
        if typ in ['station', 'zstation', 'sid', 'networkselect']:
            # A bit of hackery here if we have a name ending in a number
            netname = "network%s" % (name[-1] if name[-1] != 'n' else '',)
            ctx[netname] = fdict.get(netname)
            # The network variable tags along and within a non-PHP context,
            # this variable is unset, so we do some more hackery here
            if ctx[netname] is None:
                ctx[netname] = opt.get('network')
        elif typ in ['int', 'month', 'zhour', 'hour', 'day', 'year']:
            if value is not None:
                value = int(value)
            if default is not None:
                default = int(default)
        elif typ == 'float':
            if value is not None:
                value = float(value)
            if default is not None:
                default = float(default)
        elif typ == 'select':
            options = opt.get('options', dict())
            if value not in options:
                value = default
        elif typ == 'datetime':
            # tricky here, php has YYYY/mm/dd and CGI has YYYY-mm-dd
            if default is not None:
                default = datetime.datetime.strptime(default,
                                                     '%Y/%m/%d %H%M')
            if minval is not None:
                minval = datetime.datetime.strptime(minval,
                                                    '%Y/%m/%d %H%M')
            if maxval is not None:
                maxval = datetime.datetime.strptime(maxval,
                                                    '%Y/%m/%d %H%M')
            if value is not None:
                if value.find(" ") == -1:
                    value += " 0000"
                value = datetime.datetime.strptime(value, '%Y-%m-%d %H%M')
        elif typ == 'date':
            # tricky here, php has YYYY/mm/dd and CGI has YYYY-mm-dd
            if default is not None:
                default = datetime.datetime.strptime(default,
                                                     '%Y/%m/%d').date()
            if minval is not None:
                minval = datetime.datetime.strptime(minval,
                                                    '%Y/%m/%d').date()
            if maxval is not None:
                maxval = datetime.datetime.strptime(maxval,
                                                    '%Y/%m/%d').date()
            if value is not None:
                value = datetime.datetime.strptime(value, '%Y-%m-%d').date()
        elif typ == 'vtec_ps':
            # VTEC phenomena and significance
            for label in ['phenomena', 'significance']:
                label = label + name
                ctx[label] = fdict.get(label)
            continue
        # validation
        if minval is not None and value is not None and value < minval:
            value = default
        if maxval is not None and value is not None and value > maxval:
            value = default
        ctx[name] = value if value is not None else default
    return ctx

def exponential_backoff(func, *args, **kwargs):
    """ Exponentially backoff some function until it stops erroring"""
    msgs = []
    for i in range(5):
        try:
            return func(*args, **kwargs)
        except socket_error as serr:
            msgs.append("%s/5 %s traceback: %s" % (i+1, func.__name__, serr))
            time.sleep((2 ** i) + (random.randint(0, 1000) / 1000))
        except Exception as exp:
            msgs.append("%s/5 %s traceback: %s" % (i+1, func.__name__, exp))
            time.sleep((2 ** i) + (random.randint(0, 1000) / 1000))
    logging.error("%s failure" % (func.__name__,))
    logging.error("\n".join(msgs))
    return None

def send2box(filenames, remote_path, remotenames=None,
             ftpserver='ftp.box.com', tmpdir='/tmp', fs=None):
    """Send one or more files to CyBox

    Box has a filesize limit of 15 GB, so if we find any files larger than
    that, we shall split them into chunks prior to uploading.

    Args:
      filenames (str or list): filenames to upload
      remote_path (str): location to place the filenames
      remotenames (str or list): filenames to use on the remote FTP server
        should match size and type of filenames
      ftpserver (str): FTP server to connect to...
      tmpdir (str, optional): Temperary folder to if an individual file is over
        15 GB in size

    Returns:
      FTPSession
      list of success `True` or failures `False` matching filenames
    """
    credentials = netrc.netrc().hosts[ftpserver]
    if fs is None:
        fs = FTPSession(ftpserver, credentials[0], credentials[2],
                        tmpdir=tmpdir)
    if isinstance(filenames, str):
        filenames = [filenames, ]
    if remotenames is None:
        remotenames = filenames
    if isinstance(remotenames, str):
        remotenames = [remotenames, ]
    res = fs.put_files(remote_path, filenames, remotenames)
    return fs, res

def get_properties():
    """Fetch the properties set

    Returns:
      dict: a dictionary of property names and values (both str)
    """
    pgconn = psycopg2.connect(database='mesosite', host='iemdb', user='nobody')
    cursor = pgconn.cursor()
    cursor.execute("""SELECT propname, propvalue from properties""")
    res = {}
    for row in cursor:
        res[row[0]] = row[1]
    return res

def drct2text(drct):
    """Convert an degree value to text representation of direction.

    Args:
      drct (int or float): Value in degrees to convert to text

    Returns:
      str: String representation of the direction, could be `None`

    """
    if drct is None:
        return None
    # Convert the value into a float
    drct = float(drct)
    if drct > 360:
        return None
    text = None
    if drct >= 350 or drct < 13:
        text = "N"
    elif drct >= 13 and drct < 35:
        text = "NNE"
    elif drct >= 35 and drct < 57:
        text = "NE"
    elif drct >= 57 and drct < 80:
        text = "ENE"
    elif drct >= 80 and drct < 102:
        text = "E"
    elif drct >= 102 and drct < 127:
        text = "ESE"
    elif drct >= 127 and drct < 143:
        text = "SE"
    elif drct >= 143 and drct < 166:
        text = "SSE"
    elif drct >= 166 and drct < 190:
        text = "S"
    elif drct >= 190 and drct < 215:
        text = "SSW"
    elif drct >= 215 and drct < 237:
        text = "SW"
    elif drct >= 237 and drct < 260:
        text = "WSW"
    elif drct >= 260 and drct < 281:
        text = "W"
    elif drct >= 281 and drct < 304:
        text = "WNW"
    elif drct >= 304 and drct < 324:
        text = "NW"
    elif drct >= 324 and drct < 350:
        text = "NNW"
    return text

if __name__ == '__main__':
    logger = logging.getLogger()
    logger.setLevel(logging.DEBUG)
    send2box(['util.py', 'plot.py'], '/bah1/bah2/', remotenames=['util2.py',
                                                                 'plot.py'])
    # mirror2box("/tmp/mytest", "mytest")
<|endoftext|>


<commit_before>import enum

import tensorflow as tf

from . import util
from .flag import FLAGS, add_flag, add_required_flag

MODES = [tf.contrib.learn.ModeKeys.TRAIN,
         tf.contrib.learn.ModeKeys.EVAL,
         tf.contrib.learn.ModeKeys.INFER]

def _add_file_flag(mode):
    assert isinstance(mode, str)

    flag_name = "{}_file".format(mode)
    add_required_flag(flag_name,
                      help="File path of {0} data file(s). "
                           "A glob is available. (e.g. {0}/*.tfrecords)"
                           .format(mode))
    return flag_name

def def_def_def_input_fn(mode):
    assert mode in MODES

    BATCH_SIZE = 64

    def def_def_input_fn(batch_inputs=True, prepare_filename_queues=True):
        if batch_inputs:
            add_flag("batch_size", type=int, default=BATCH_SIZE,
                     help="Mini-batch size")
            add_flag("batch_queue_capacity", type=int, default=BATCH_SIZE * 16,
                     help="Batch queue capacity")

        if prepare_filename_queues:
            file_flag = _add_file_flag(mode)
            filenames_to_queue = def_filenames_to_queue(mode)

        def def_input_fn(user_input_fn):
            @util.func_scope
            def input_fn():
                if prepare_filename_queues:
                    x, y = user_input_fn(filenames_to_queue({
                        mode: tf.train.match_filenames_once(
                            getattr(FLAGS, "{}_file".format(mode)),
                            name="{}_filenames".format(mode))
                        for mode in [tf.contrib.learn.ModeKeys.TRAIN,
                                     tf.contrib.learn.ModeKeys.EVAL]}[mode]))
                else:
                    x, y = user_input_fn()

                if not batch_inputs:
                    return x, y

                tuple_input = isinstance(x, tf.Tensor)

                if not tuple_input:
                    duplicate_keys = x.keys() & y.keys()
                    if len(duplicate_keys) != 0:
                        raise ValueError(
                            "Some keys of x and y are duplicate. ({})"
                            .format(duplicate_keys))

                inputs = (tf.train.shuffle_batch
                          if mode == tf.contrib.learn.ModeKeys.TRAIN else
                          tf.train.batch)(
                    [x, y] if tuple_input else {**x, **y},
                    batch_size=FLAGS.batch_size,
                    capacity=FLAGS.batch_queue_capacity,
                    **({"min_after_dequeue": FLAGS.batch_queue_capacity // 2}
                       if mode == tf.contrib.learn.ModeKeys.TRAIN else
                       {"allow_smaller_final_batch": True}))

                restore = lambda x: {key: inputs[key] for key in x.keys()}

                return inputs if tuple_input else (restore(x), restore(y))

            return input_fn

        return def_input_fn

    return def_def_input_fn

def_def_train_input_fn = def_def_def_input_fn(tf.contrib.learn.ModeKeys.TRAIN)
def_def_eval_input_fn = def_def_def_input_fn(tf.contrib.learn.ModeKeys.EVAL)

def def_filenames_to_queue(mode):
    assert mode in MODES

    add_flag("filename_queue_capacity", type=int, default=32,
             help="Capacity of filename queues of {}, {} and {} data"
                  .format(*MODES))

    @util.func_scope
    def filenames_to_queue(filenames):
        return tf.train.string_input_producer(
            filenames,
            num_epochs=(None
                        if mode == tf.contrib.learn.ModeKeys.TRAIN else
                        1),
            shuffle=(mode == tf.contrib.learn.ModeKeys.TRAIN),
            capacity=FLAGS.filename_queue_capacity)

    return filenames_to_queue
<commit_msg>Define def_def_infer_input_fn<commit_after>import enum

import tensorflow as tf

from . import util
from .flag import FLAGS, add_flag, add_required_flag

MODES = [tf.contrib.learn.ModeKeys.TRAIN,
         tf.contrib.learn.ModeKeys.EVAL,
         tf.contrib.learn.ModeKeys.INFER]

def _add_file_flag(mode):
    assert isinstance(mode, str)

    flag_name = "{}_file".format(mode)
    add_required_flag(flag_name,
                      help="File path of {0} data file(s). "
                           "A glob is available. (e.g. {0}/*.tfrecords)"
                           .format(mode))
    return flag_name

def def_def_def_input_fn(mode):
    assert mode in MODES

    BATCH_SIZE = 64

    def def_def_input_fn(batch_inputs=True, prepare_filename_queues=True):
        if batch_inputs:
            add_flag("batch_size", type=int, default=BATCH_SIZE,
                     help="Mini-batch size")
            add_flag("batch_queue_capacity", type=int, default=BATCH_SIZE * 16,
                     help="Batch queue capacity")

        if prepare_filename_queues:
            file_flag = _add_file_flag(mode)
            filenames_to_queue = def_filenames_to_queue(mode)

        def def_input_fn(user_input_fn):
            @util.func_scope
            def input_fn():
                if prepare_filename_queues:
                    x, y = user_input_fn(filenames_to_queue({
                        mode: tf.train.match_filenames_once(
                            getattr(FLAGS, "{}_file".format(mode)),
                            name="{}_filenames".format(mode))
                        for mode in [tf.contrib.learn.ModeKeys.TRAIN,
                                     tf.contrib.learn.ModeKeys.EVAL]}[mode]))
                else:
                    x, y = user_input_fn()

                if not batch_inputs:
                    return x, y

                tuple_input = isinstance(x, tf.Tensor)

                if not tuple_input:
                    duplicate_keys = x.keys() & y.keys()
                    if len(duplicate_keys) != 0:
                        raise ValueError(
                            "Some keys of x and y are duplicate. ({})"
                            .format(duplicate_keys))

                inputs = (tf.train.shuffle_batch
                          if mode == tf.contrib.learn.ModeKeys.TRAIN else
                          tf.train.batch)(
                    [x, y] if tuple_input else {**x, **y},
                    batch_size=FLAGS.batch_size,
                    capacity=FLAGS.batch_queue_capacity,
                    **({"min_after_dequeue": FLAGS.batch_queue_capacity // 2}
                       if mode == tf.contrib.learn.ModeKeys.TRAIN else
                       {"allow_smaller_final_batch": True}))

                restore = lambda x: {key: inputs[key] for key in x.keys()}

                return inputs if tuple_input else (restore(x), restore(y))

            return input_fn

        return def_input_fn

    return def_def_input_fn

for mode in MODES:
    globals()["def_def_{}_input_fn".format(mode)] = def_def_def_input_fn(mode)

def def_filenames_to_queue(mode):
    assert mode in MODES

    add_flag("filename_queue_capacity", type=int, default=32,
             help="Capacity of filename queues of {}, {} and {} data"
                  .format(*MODES))

    @util.func_scope
    def filenames_to_queue(filenames):
        return tf.train.string_input_producer(
            filenames,
            num_epochs=(None
                        if mode == tf.contrib.learn.ModeKeys.TRAIN else
                        1),
            shuffle=(mode == tf.contrib.learn.ModeKeys.TRAIN),
            capacity=FLAGS.filename_queue_capacity)

    return filenames_to_queue
<|endoftext|>


<commit_before>import logging
import sys
import time
import Queue

import Backend
import Devices
import Interfaces

class Flow:
   """ Holds all the state of a flow/pour """
   def __init__(self, channel, start = None, user = None,
         ticks = 0, max_volume = sys.maxint, end = None):
      self.channel = channel
      self.start = start or time.time()
      self.end = end or start
      self.user = user
      self._ticks = ticks
      self.max_volume = max_volume

      self._last_activity = time.time()
      self._last_ticks = None

   def SetTicks(self, ticks):
      """ ticks should be a monotonically increasing count """
      if self._last_ticks is None:
         self._last_ticks = ticks

      diff = ticks - self._last_ticks
      self._ticks += diff
      self._last_ticks = ticks
      if diff != 0:
         self._last_activity = time.time()
      return diff

   def IdleSeconds(self):
      return time.time() - self._last_activity

   def Ticks(self):
      return self._ticks

   def Keg(self):
      return self.channel.GetKeg()

class Channel:
   """
   A Channel is a path of beer contianing a flow controller and drinker queue.

   For example, the typical kegerator has only one keg and thus one channel. A
   4-tap kegbot would have 4 channels, each channel having its own flow
   controller instance and drinker "flow" queue.
   """
   def __init__(self, chanid, valve_relay = None, flow_meter = None):
      self.chanid = chanid
      self.logger = logging.getLogger('channel%s' % str(chanid))

      if valve_relay is None:
         valve_relay = Devices.NoOp.Relay()
      assert isinstance(valve_relay,Interfaces.IRelay), \
            "valve_relay must implement IRelay interface"
      self.valve_relay = valve_relay

      if flow_meter is None:
         flow_meter = Devices.NoOp.Flowmeter()
      assert isinstance(flow_meter, Interfaces.IFlowmeter),\
            "flow_meter must implement IFlowmeter interface"
      self.flow_meter = flow_meter

      self.flow_queue = Queue.Queue()
      self.active_flow = None

   def IsIdle(self):
      return self.active_flow is None

   def EnqueueFlow(self, flow):
      """ Add a flow to the waiting queue of flows """
      self.flow_queue.put(flow)

   def MaybeActivateNextFlow(self):
      """ If there isn't an active flow, pop one from flow_queue and activate """
      if self.active_flow is not None:
         return None
      try:
         flow = self.flow_queue.get_nowait()
      except Queue.Empty:
         return None
      self.active_flow = flow
      return flow

   def EnableValve(self):
      return self.valve_relay.Enable()

   def DisableValve(self):
      return self.valve_relay.Disable()

   def GetTicks(self):
      return self.flow_meter.GetTicks()

   def DeactivateFlow(self):
      """ Reset active_flow state variable to None """
      self.active_flow = None

   def GetKeg(self):
      online_kegs = list(Backend.Keg.selectBy(status='online',
         channel=self.chanid, orderBy='-id'))
      if len(online_kegs) == 0:
         return None
      else:
         return online_kegs[0]

<commit_msg>- more properly define Flow error; there should be exactly 1 online keg per   channel<commit_after>import logging
import sys
import time
import Queue

import Backend
import Devices
import Interfaces

class Flow:
   """ Holds all the state of a flow/pour """
   def __init__(self, channel, start = None, user = None,
         ticks = 0, max_volume = sys.maxint, end = None):
      self.channel = channel
      self.start = start or time.time()
      self.end = end or start
      self.user = user
      self._ticks = ticks
      self.max_volume = max_volume

      self._last_activity = time.time()
      self._last_ticks = None

   def SetTicks(self, ticks):
      """ ticks should be a monotonically increasing count """
      if self._last_ticks is None:
         self._last_ticks = ticks

      diff = ticks - self._last_ticks
      self._ticks += diff
      self._last_ticks = ticks
      if diff != 0:
         self._last_activity = time.time()
      return diff

   def IdleSeconds(self):
      return time.time() - self._last_activity

   def Ticks(self):
      return self._ticks

   def Keg(self):
      return self.channel.GetKeg()

class Channel:
   """
   A Channel is a path of beer contianing a flow controller and drinker queue.

   For example, the typical kegerator has only one keg and thus one channel. A
   4-tap kegbot would have 4 channels, each channel having its own flow
   controller instance and drinker "flow" queue.
   """
   def __init__(self, chanid, valve_relay = None, flow_meter = None):
      self.chanid = chanid
      self.logger = logging.getLogger('channel%s' % str(chanid))

      if valve_relay is None:
         valve_relay = Devices.NoOp.Relay()
      assert isinstance(valve_relay,Interfaces.IRelay), \
            "valve_relay must implement IRelay interface"
      self.valve_relay = valve_relay

      if flow_meter is None:
         flow_meter = Devices.NoOp.Flowmeter()
      assert isinstance(flow_meter, Interfaces.IFlowmeter),\
            "flow_meter must implement IFlowmeter interface"
      self.flow_meter = flow_meter

      self.flow_queue = Queue.Queue()
      self.active_flow = None

   def IsIdle(self):
      return self.active_flow is None

   def EnqueueFlow(self, flow):
      """ Add a flow to the waiting queue of flows """
      self.flow_queue.put(flow)

   def MaybeActivateNextFlow(self):
      """ If there isn't an active flow, pop one from flow_queue and activate """
      if self.active_flow is not None:
         return None
      try:
         flow = self.flow_queue.get_nowait()
      except Queue.Empty:
         return None
      self.active_flow = flow
      return flow

   def EnableValve(self):
      return self.valve_relay.Enable()

   def DisableValve(self):
      return self.valve_relay.Disable()

   def GetTicks(self):
      return self.flow_meter.GetTicks()

   def DeactivateFlow(self):
      """ Reset active_flow state variable to None """
      self.active_flow = None

   def GetKeg(self):
      channel_kegs = list(Backend.Keg.selectBy(status='online',
         channel=self.chanid, orderBy='-id'))
      if len(channel_kegs) != 1:
         return None
      return channel_kegs[0]

<|endoftext|>


<commit_before>'''
Created on Mar 7, 2011

@author: johnsalvatier
'''
from point import *
from types import *

import theano
import theano.tensor as t

from theano import function, scan

import numpy as np

__all__ = ['Model', 'logp', 'dlogp', 'continuous_vars']

def Variable(name, shape, dtype, testval):
    """
    Creates a TensorVariable of the given shape and type

    Parameters
    ----------

    shape : int or vector of ints
    dtype : str

    Examples
    --------

    """
    shape = np.atleast_1d(shape)
    var = t.TensorType(str(dtype), shape == 1)(name)
    var.dshape = tuple(shape)
    var.dsize = int(np.prod(shape))
    var.tag.test_value = np.ones(shape, dtype)*testval

    return var

val_defaults = {'discrete'   : ['mode'],
                'continuous' : ['median', 'mean', 'mode']}

def try_defaults(dist):
    vals =  val_defaults[dist.support]

    for val in vals:
        if hasattr(dist, val):
            return getattr(dist,val)
    raise AttributeError(str(dist) + " does not have a value for any of: " + str(vals))

class Model(object):
    """
    Base class for encapsulation of the variables and
    likelihood factors of a model.
    """

    def __init__(self):
       self.vars = []
       self.factors = []

    """
    these functions add random variables
    it is totally appropriate to add new functions to Model
    """
    def Data(model, data, distribution):
        args = map(t.constant, as_iterargs(data))
        model.factors.append(distribution.logp(*args))

    def Var(model, name, distribution, shape = 1, dtype = None, testval = try_defaults):
        if not dtype:
            dtype = default_type[distribution.support]

        var = Variable(name, shape, dtype, get_test_val(distribution, testval))

        model.vars.append(var)
        model.factors.append(distribution.logp(var))
        return var

    def fn(self, calc, mode = None):
        if hasattr(calc, '__call__'):
            out = calc(self)
        else:
            out = [f(self) for f in calc]

        return PointFunc(
                function(self.vars,out,
                    allow_input_downcast = True, mode = mode))

    def logp(self):
        return self.fn(logp)

    def dlogp(self, *args, **kwargs):
        return self.fn(dlogp(*args, **kwargs))

    @property
    def test_point(self):
        return dict( (str(var), var.tag.test_value) for var in self.vars)

def as_iterargs(data):
    if isinstance(data, tuple):
        return data
    if hasattr(data, 'columns'): #data frames
        return [np.asarray(data[c]) for c in data.columns]
    else:
        return [data]

def get_test_val(dist, val):
    try :
        val = getattr(dist, val)
    except TypeError:
        pass

    if hasattr(val, '__call__'):
        val = val(dist)

    if isinstance(val, t.TensorVariable):
        return val.tag.test_value
    else:
        return val

def continuous_vars(model):
    return [ var for var in model.vars if var.dtype in continuous_types]

class PointFunc(object):
    def __init__(self, f):
        self.f = f
    def __call__(self,state):
        return self.f(**state)

"""
These functions build log-posterior graphs (and derivatives)
"""
def logp(model):
    """
    Calculates the log-probability of a specified model

    Parameters
    ----------

    model : Model

    Examples
    --------

    >>> an example

    """
    return t.add(*map(t.sum, model.factors))

def flatgrad(f, v):
    return t.flatten(t.grad(f, v))

def gradient(f, dvars):
    return t.concatenate([flatgrad(f, v) for v in dvars])
def jacobian(f, dvars):
    def jac(v):
        def grad_i(i, f1, v):
            return flatgrad(f1[i], v)

        return scan(grad_i, sequences=t.arange(f.shape[0]), non_sequences=[f,v])[0]

    return t.concatenate(map(jac, dvars))

def hessian(f, dvars):
    return jacobian(gradient(f, dvars), dvars)

def dlogp(dvars=None, n=1):
    """
    Returns a function for calculating the derivative of the output
    of another function.

    Parameters
    ----------
    d_calc : function

    Returns
    -------
    der_calc : function
    """

    dfn = [gradient, hessian]
    def dlogp_calc(model):

        if dvars is None:
            vars = continuous_vars(model)
        else:
            vars = dvars
        return dfn[n-1](logp(model), vars)

    return dlogp_calc

#theano stuff
theano.config.warn.sum_div_dimshuffle_bug = False
theano.config.compute_test_value = 'raise'
<commit_msg>refactored compilation of model functions<commit_after>'''
Created on Mar 7, 2011

@author: johnsalvatier
'''
from point import *
from types import *

from theano import theano, tensor as t, function
from theano.gof.graph import inputs

import numpy as np

__all__ = ['Model', 'compilef', 'gradient', 'hessian']

def Variable(name, shape, dtype, testval):
    """
    Creates a TensorVariable of the given shape and type

    Parameters
    ----------

    shape : int or vector of ints
    dtype : str

    Examples
    --------

    """
    shape = np.atleast_1d(shape)
    var = t.TensorType(str(dtype), shape == 1)(name)
    var.dshape = tuple(shape)
    var.dsize = int(np.prod(shape))
    var.tag.test_value = np.ones(shape, dtype)*testval

    return var

val_defaults = {'discrete'   : ['mode'],
                'continuous' : ['median', 'mean', 'mode']}

def try_defaults(dist):
    vals =  val_defaults[dist.support]

    for val in vals:
        if hasattr(dist, val):
            return getattr(dist,val)
    raise AttributeError(str(dist) + " does not have a value for any of: " + str(vals))

class Model(object):
    """
    Base class for encapsulation of the variables and
    likelihood factors of a model.
    """

    def __init__(self):
       self.vars = []
       self.factors = []

    """
    these functions add random variables
    """
    def Data(model, data, dist):
        args = map(t.constant, as_iterargs(data))
        model.factors.append(dist.logp(*args))

    def Var(model, name, dist, shape = 1, dtype = None, testval = try_defaults):
        if not dtype:
            dtype = default_type[dist.support]

        var = Variable(name, shape, dtype, get_test_val(dist, testval))

        model.vars.append(var)
        model.factors.append(dist.logp(var))
        return var

    def TransformedVar(model, name, dist, transform, logjacobian, shape = 1, dtype = None, testval = try_defaults):
        if not dtype:
            dtype = default_type[dist.support]

        var = Variable('transformed_' + name, shape, dtype, get_test_val(dist, testval))

        model.vars.append(var)

        tvar = transform(var)
        model.factors.append(dist.logp(tvar) + logjacobian(var))

        return tvar, var

    @property
    def logp(model):
        """
        log-probability of the model

        Parameters
        ----------

        model : Model

        Returns
        -------

        logp : Theano scalar

        """
        return t.add(*map(t.sum, model.factors))

    @property
    def logpc(model):
        return compilef(model.logp)

    def dlogpc(model, vars = None):
        return compilef(gradient(model.logp, vars))

    def d2logpc(model, vars = None):
        return compilef(hessian(model.logp, vars))

    @property
    def test_point(self):
        return dict( (str(var), var.tag.test_value) for var in self.vars)

    @property
    def cont_vars(model):
        return typefilter(model.vars, continuous_types)

def compilef(outs, mode = None):
    return PointFunc(
                function(inputvars(outs), outs,
                         allow_input_downcast = True,
                         on_unused_input = 'ignore',
                         mode = mode)
           )

def as_iterargs(data):
    if isinstance(data, tuple):
        return data
    if hasattr(data, 'columns'): #data frames
        return [np.asarray(data[c]) for c in data.columns]
    else:
        return [data]

def get_test_val(dist, val):
    try :
        val = getattr(dist, val)
    except TypeError:
        pass

    if hasattr(val, '__call__'):
        val = val(dist)

    if isinstance(val, t.TensorVariable):
        return val.tag.test_value
    else:
        return val

def makeiter(a):
    if isinstance(a, (tuple, list)):
        return a
    else :
        return [a]

def inputvars(a):
    return [v for v in inputs(makeiter(a)) if isinstance(v, t.TensorVariable)]

"""
Theano derivative functions
"""

def cont_inputs(f):
    return typefilter(inputvars(f), continuous_types)

def gradient1(f, v):
    """flat gradient of f wrt v"""
    return t.flatten(t.grad(f, v))

def gradient(f, vars = None):
    if not vars:
        vars = cont_inputs(f)

    return t.concatenate([gradient1(f, v) for v in vars], axis = 0)

def jacobian1(f, v):
    """jacobian of f wrt v"""
    f = t.flatten(f)
    idx = t.arange(f.shape[0])

    def grad_i(i):
        return gradient1(f[i], v)

    return theano.map(grad_i, idx)[0]

def jacobian(f, vars = None):
    if not vars:
        vars = cont_inputs(f)

    return t.concatenate([jacobian1(f, v) for v in vars], axis = 1)

def hessian(f, vars = None):
    return -jacobian(gradient(f, vars), vars)

#theano stuff
theano.config.warn.sum_div_dimshuffle_bug = False
theano.config.compute_test_value = 'raise'
<|endoftext|>