<commit_before>import json
import os.path
import subprocess
import sys
import time
import urllib2

## Generated files ##

graphFile = "wk_progress.png"
resultFile = "result.txt"
apiKeyFile = "api_key.txt"
storedDataFile = "wkstats.dat"
plotDataFile = "wkstats_plot.tmp"

## Api Key related methods ##

def getApiKey():
    if os.path.isfile(apiKeyFile) is False:
        createApiFile(apiKeyFile)
        apiKeyNotFound(apiKeyFile)

    with open(apiKeyFile, "r") as f:
        for line in f:
            if line.startswith("#") or len(line.strip()) != 32:
                continue
            return line.strip()

    apiKeyNotFound(apiKeyFile)

def createApiFile(file):
    with open(file, "w") as f:
        f.write("# Please paste your WaniKani API key on the line below, you\n")
        f.write("# can find it on the account setings page on WaniKani.\n")

def apiKeyNotFound(file):
    print "API key not found. Please enter your WaniKani api key in the file " + file
    exit()

## WaniKani interaction ##

def processWaniKaniResponse(data):
    if "error" in data:
        print "WaniKani returned an error: " + data["error"]["message"]
        exit()

    ret = {}
    ret["date_saved"] = time.strftime("%Y-%m-%d")
    ret["level"] = data["user_information"]["level"]
    ret["requested_information"] = data["requested_information"]
    return ret

def readFromWaniKani():
    api_key = getApiKey()
    url = "https://www.wanikani.com/api/user/" + api_key + "/srs-distribution"
    j = json.loads(urllib2.urlopen(url).read())
    return processWaniKaniResponse(j)

## Data handling ##

def oneCell(new, old):
    if old == new:
        return "{:d}".format(new)
    else:
        return "{:d} ({:+d})".format(new, new - old)

def getLine(thisTime, lastTime, which):
    a = oneCell(thisTime["apprentice"][which], lastTime["apprentice"][which])
    g = oneCell(thisTime["guru"][which], lastTime["guru"][which])
    m = oneCell(thisTime["master"][which], lastTime["master"][which])
    e = oneCell(thisTime["enlighten"][which], lastTime["enlighten"][which])
    b = oneCell(thisTime["burned"][which], lastTime["burned"][which])
    return which.capitalize() + "|" + a +"|" + g + "|" + m + "|" + e + "|" + b

def getTableHeader(thisTime, lastTime):
    return "**Lvl " + oneCell(thisTime["level"], lastTime["level"]) + "**|**Apprentice**|**Guru**|**Master**|**Enlightened**|**Burned**"

def makeBold(line):
    def bold(word):
        return "**" + word + "**"

    return "|".join(map(bold, line.split("|")))

def getResult(thisTime, lastTime):
    key = "requested_information"
    ret = []
    ret.append(getTableHeader(thisTime, lastTime))
    ret.append("--|--:|--:|--:|--:|--:")
    ret.append(getLine(thisTime[key], lastTime[key], "radicals"))
    ret.append(getLine(thisTime[key], lastTime[key], "kanji"))
    ret.append(getLine(thisTime[key], lastTime[key], "vocabulary"))
    ret.append(makeBold(getLine(thisTime[key], lastTime[key], "total")))
    return ret;

def getBurnedCount(data, what):
    return data["requested_information"]["burned"][what]

def getTotalCount(data, what):
    return data["requested_information"]["apprentice"][what] + data["requested_information"]["guru"][what] + data["requested_information"]["master"][what] + data["requested_information"]["enlighten"][what] + getBurnedCount(data, what)

## File handling ##

def storeData(data):
    lines = []
    lastLine = None
    if os.path.isfile(storedDataFile) is not False:
        with open(storedDataFile, "r") as f:
            lines = f.readlines()
        lastLine = json.loads(lines[-1])

    with open(storedDataFile, "w") as f:
        if lastLine is not None and lastLine["date_saved"] == data["date_saved"]:
            f.writelines(lines[:-1])
        else:
            f.writelines(lines)

        json.dump(data, f, sort_keys = True)
        f.write("\n")

def getLastEntry():
    if os.path.isfile(storedDataFile) is False:
        return None

    with open(storedDataFile, "r") as f:
        for line in f:
            pass
        return json.loads(line)

# result is a list of lines to write to the file
def printResultToFile(result):
    with open(resultFile, "w") as f:
        output = "\n".join(result)
        f.write(output)
        print "Reddit table with latest data written to: " + resultFile

# returns true if plot data was successfully written to file
def writePlotData(thisTime):
    if os.path.isfile(storedDataFile) is False:
        print "Warning: No previously stored data found, unable to plot a graph."
        return False

    weeks = []
    with open(storedDataFile, "r") as f:
        weeks = f.readlines()

    if thisTime is not None:
        # thisTime is None if the result was stored to file
        weeks.append(thisTime)

    if len(weeks) == 1:
        print "Warning: Only one stored entry found, unable to plot a graph."
        return False

    with open(plotDataFile, "w") as f:
        for week in weeks:
            data = json.loads(week)
            rburn = getBurnedCount(data, "radicals")
            rtot = getTotalCount(data, "radicals")
            kburn = getBurnedCount(data, "kanji")
            ktot = getTotalCount(data, "kanji")
            vburn = getBurnedCount(data, "vocabulary")
            vtot = getTotalCount(data, "vocabulary")
            level = data["level"]

            f.write(data["date_saved"] + " " + str(rburn) + " " + str(rtot) + " " + str(kburn) + " " + str(ktot) + " " + str(vburn) + " " + str(vtot) + " " + str(level) + "\n")
    return True

## Graph plotting ##

def plotGraph():
    command = "gnuplot wkstats.p"
    subprocess.call(command.split())
    print "Graph of WaniKani progress written to: " + graphFile

def deletePlotData():
    os.remove(plotDataFile)

def makeGraph(thisTime):
    if writePlotData(thisTime) is True:
        plotGraph()
        deletePlotData()

## Main handling ##

def incorrectArguments(args):
    print "Incorrect usage: " + " ".join(args)
    print "Supported options:"
    print " --save: Saves the data read from WaniKani to file. If the last entry"
    print "         in the file is from today it will be replaced with the new"
    print "         data from WaniKani."

def wkstats():
    shouldSave = False
    if len(sys.argv) > 1 and sys.argv[1] == "--save":
        shouldSave = True
    elif len(sys.argv) > 1:
        incorrectArguments(sys.argv)
        exit()

    thisTime = readFromWaniKani()
    lastTime = getLastEntry()
    if lastTime is None:
        lastTime = thisTime

    thisTimeString = None
    if shouldSave:
        storeData(thisTime)
    else:
        thisTimeString = json.dumps(thisTime)

    result = getResult(thisTime, lastTime)
    printResultToFile(result)
    makeGraph(thisTimeString)

if __name__ == "__main__":
    wkstats();

''' Example response from WaniKani:
{
  "user_information": {
    "username": "nibarius",
    "gravatar": "4dff6dbd5d9ec926ffb5266be887526f",
    "level": 25,
    "title": "Turtles",
    "about": "",
    "website": null,
    "twitter": "",
    "topics_count": 9,
    "posts_count": 101,
    "creation_date": 1355330482,
    "vacation_date": null
  },
  "requested_information": {
    "apprentice": {
      "radicals": 8,
      "kanji": 36,
      "vocabulary": 56,
      "total": 100
    },
    "guru": {
      "radicals": 13,
      "kanji": 76,
      "vocabulary": 257,
      "total": 346
    },
    "master": {
      "radicals": 15,
      "kanji": 96,
      "vocabulary": 245,
      "total": 356
    },
    "enlighten": {
      "radicals": 56,
      "kanji": 220,
      "vocabulary": 552,
      "total": 828
    },
    "burned": {
      "radicals": 250,
      "kanji": 404,
      "vocabulary": 1272,
      "total": 1926
    }
  }
}
'''
<commit_msg>Make a printout when saving to file.<commit_after>import json
import os.path
import subprocess
import sys
import time
import urllib2

## Generated files ##

graphFile = "wk_progress.png"
resultFile = "result.txt"
apiKeyFile = "api_key.txt"
storedDataFile = "wkstats.dat"
plotDataFile = "wkstats_plot.tmp"

## Api Key related methods ##

def getApiKey():
    if os.path.isfile(apiKeyFile) is False:
        createApiFile(apiKeyFile)
        apiKeyNotFound(apiKeyFile)

    with open(apiKeyFile, "r") as f:
        for line in f:
            if line.startswith("#") or len(line.strip()) != 32:
                continue
            return line.strip()

    apiKeyNotFound(apiKeyFile)

def createApiFile(file):
    with open(file, "w") as f:
        f.write("# Please paste your WaniKani API key on the line below, you\n")
        f.write("# can find it on the account setings page on WaniKani.\n")

def apiKeyNotFound(file):
    print "API key not found. Please enter your WaniKani api key in the file " + file
    exit()

## WaniKani interaction ##

def processWaniKaniResponse(data):
    if "error" in data:
        print "WaniKani returned an error: " + data["error"]["message"]
        exit()

    ret = {}
    ret["date_saved"] = time.strftime("%Y-%m-%d")
    ret["level"] = data["user_information"]["level"]
    ret["requested_information"] = data["requested_information"]
    return ret

def readFromWaniKani():
    api_key = getApiKey()
    url = "https://www.wanikani.com/api/user/" + api_key + "/srs-distribution"
    j = json.loads(urllib2.urlopen(url).read())
    return processWaniKaniResponse(j)

## Data handling ##

def oneCell(new, old):
    if old == new:
        return "{:d}".format(new)
    else:
        return "{:d} ({:+d})".format(new, new - old)

def getLine(thisTime, lastTime, which):
    a = oneCell(thisTime["apprentice"][which], lastTime["apprentice"][which])
    g = oneCell(thisTime["guru"][which], lastTime["guru"][which])
    m = oneCell(thisTime["master"][which], lastTime["master"][which])
    e = oneCell(thisTime["enlighten"][which], lastTime["enlighten"][which])
    b = oneCell(thisTime["burned"][which], lastTime["burned"][which])
    return which.capitalize() + "|" + a +"|" + g + "|" + m + "|" + e + "|" + b

def getTableHeader(thisTime, lastTime):
    return "**Lvl " + oneCell(thisTime["level"], lastTime["level"]) + "**|**Apprentice**|**Guru**|**Master**|**Enlightened**|**Burned**"

def makeBold(line):
    def bold(word):
        return "**" + word + "**"

    return "|".join(map(bold, line.split("|")))

def getResult(thisTime, lastTime):
    key = "requested_information"
    ret = []
    ret.append(getTableHeader(thisTime, lastTime))
    ret.append("--|--:|--:|--:|--:|--:")
    ret.append(getLine(thisTime[key], lastTime[key], "radicals"))
    ret.append(getLine(thisTime[key], lastTime[key], "kanji"))
    ret.append(getLine(thisTime[key], lastTime[key], "vocabulary"))
    ret.append(makeBold(getLine(thisTime[key], lastTime[key], "total")))
    return ret;

def getBurnedCount(data, what):
    return data["requested_information"]["burned"][what]

def getTotalCount(data, what):
    return data["requested_information"]["apprentice"][what] + data["requested_information"]["guru"][what] + data["requested_information"]["master"][what] + data["requested_information"]["enlighten"][what] + getBurnedCount(data, what)

## File handling ##

def storeData(data):
    lines = []
    lastLine = None
    if os.path.isfile(storedDataFile) is not False:
        with open(storedDataFile, "r") as f:
            lines = f.readlines()
        lastLine = json.loads(lines[-1])

    with open(storedDataFile, "w") as f:
        if lastLine is not None and lastLine["date_saved"] == data["date_saved"]:
            f.writelines(lines[:-1])
        else:
            f.writelines(lines)

        json.dump(data, f, sort_keys = True)
        f.write("\n")
    print "Latest data saved to: " + storedDataFile

def getLastEntry():
    if os.path.isfile(storedDataFile) is False:
        return None

    with open(storedDataFile, "r") as f:
        for line in f:
            pass
        return json.loads(line)

# result is a list of lines to write to the file
def printResultToFile(result):
    with open(resultFile, "w") as f:
        output = "\n".join(result)
        f.write(output)
        print "Reddit table with latest data written to: " + resultFile

# returns true if plot data was successfully written to file
def writePlotData(thisTime):
    if os.path.isfile(storedDataFile) is False:
        print "Warning: No previously stored data found, unable to plot a graph."
        return False

    weeks = []
    with open(storedDataFile, "r") as f:
        weeks = f.readlines()

    if thisTime is not None:
        # thisTime is None if the result was stored to file
        weeks.append(thisTime)

    if len(weeks) == 1:
        print "Warning: Only one stored entry found, unable to plot a graph."
        return False

    with open(plotDataFile, "w") as f:
        for week in weeks:
            data = json.loads(week)
            rburn = getBurnedCount(data, "radicals")
            rtot = getTotalCount(data, "radicals")
            kburn = getBurnedCount(data, "kanji")
            ktot = getTotalCount(data, "kanji")
            vburn = getBurnedCount(data, "vocabulary")
            vtot = getTotalCount(data, "vocabulary")
            level = data["level"]

            f.write(data["date_saved"] + " " + str(rburn) + " " + str(rtot) + " " + str(kburn) + " " + str(ktot) + " " + str(vburn) + " " + str(vtot) + " " + str(level) + "\n")
    return True

## Graph plotting ##

def plotGraph():
    command = "gnuplot wkstats.p"
    subprocess.call(command.split())
    print "Graph of WaniKani progress written to: " + graphFile

def deletePlotData():
    os.remove(plotDataFile)

def makeGraph(thisTime):
    if writePlotData(thisTime) is True:
        plotGraph()
        deletePlotData()

## Main handling ##

def incorrectArguments(args):
    print "Incorrect usage: " + " ".join(args)
    print "Supported options:"
    print " --save: Saves the data read from WaniKani to file. If the last entry"
    print "         in the file is from today it will be replaced with the new"
    print "         data from WaniKani."

def wkstats():
    shouldSave = False
    if len(sys.argv) > 1 and sys.argv[1] == "--save":
        shouldSave = True
    elif len(sys.argv) > 1:
        incorrectArguments(sys.argv)
        exit()

    thisTime = readFromWaniKani()
    lastTime = getLastEntry()
    if lastTime is None:
        lastTime = thisTime

    thisTimeString = None
    if shouldSave:
        storeData(thisTime)
    else:
        thisTimeString = json.dumps(thisTime)

    result = getResult(thisTime, lastTime)
    printResultToFile(result)
    makeGraph(thisTimeString)

if __name__ == "__main__":
    wkstats();

''' Example response from WaniKani:
{
  "user_information": {
    "username": "nibarius",
    "gravatar": "4dff6dbd5d9ec926ffb5266be887526f",
    "level": 25,
    "title": "Turtles",
    "about": "",
    "website": null,
    "twitter": "",
    "topics_count": 9,
    "posts_count": 101,
    "creation_date": 1355330482,
    "vacation_date": null
  },
  "requested_information": {
    "apprentice": {
      "radicals": 8,
      "kanji": 36,
      "vocabulary": 56,
      "total": 100
    },
    "guru": {
      "radicals": 13,
      "kanji": 76,
      "vocabulary": 257,
      "total": 346
    },
    "master": {
      "radicals": 15,
      "kanji": 96,
      "vocabulary": 245,
      "total": 356
    },
    "enlighten": {
      "radicals": 56,
      "kanji": 220,
      "vocabulary": 552,
      "total": 828
    },
    "burned": {
      "radicals": 250,
      "kanji": 404,
      "vocabulary": 1272,
      "total": 1926
    }
  }
}
'''
<|endoftext|>


<commit_before>import numpy as np
import matplotlib.pyplot as pl

import pygp
import pygp.plotting
import pybo.models
import pybo.policies

def callback(info, x, f, model, bounds, index):
    xmin = bounds[0, 0]
    xmax = bounds[0, 1]

    X = np.linspace(xmin, xmax, 500)[:, None]
    F = f.get_f(X)
    ymin, ymax = F.min(), F.max()
    ymin -= 0.2 * (ymax - ymin)
    ymax += 0.4 * (ymax - ymin)

    pygp.plotting.plot(model, xmin=xmin, xmax=xmax,
                       ymin=ymin, ymax=ymax,
                       subplot=211, draw=False)

    pl.plot(X, F, 'k--', lw=2)
    pl.axvline(x, color='r')
    pl.axvline(info[-1]['xbest'], color='g')
    pl.ylabel('posterior')
    pl.gca().set_xticklabels([])
    pl.gca().set_yticklabels([])

    pl.subplot(212)
    pl.cla()
    pl.plot(X, index(X), lw=2)
    pl.axvline(x, color='r')
    pl.axis(xmin=xmin, xmax=xmax)
    pl.ylabel('acquisition')
    pl.gca().set_xticklabels([])
    pl.gca().set_yticklabels([])
    pl.draw()

if __name__ == '__main__':
    T = 100
    sigma = 0.05
    model = pygp.BasicGP(sigma, 1.0, 0.1, kernel='matern3')
    f = pybo.models.GPModel([3, 5], model)

    pybo.policies.solve_bayesopt(f,
                                 f.bounds,
                                 sigma,
                                 policy='ucb',
                                 kernel=model._kernel.copy(),
                                 callback=callback)
<commit_msg>Clean up the demo.<commit_after>"""
Simple demo which performs Bayesian optimization on a single objective function
and visualizes the search process while doing so.
"""

import numpy as np
import matplotlib.pyplot as pl

import pygp
import pygp.plotting as pp
import pybo.models
import pybo.policies

def callback(info, x, f, model, bounds, index):
    """
    Plot the current posterior and the index.
    """
    xmin = bounds[0, 0]
    xmax = bounds[0, 1]

    X = np.linspace(xmin, xmax, 500)[:, None]
    F = f.get_f(X)
    ymin, ymax = F.min(), F.max()
    ymin -= 0.2 * (ymax - ymin)
    ymax += 0.4 * (ymax - ymin)

    pl.subplot(211)
    pl.cla()
    pp.plot_posterior(model, xmin=xmin, xmax=xmax)
    pl.axis((xmin, xmax, ymin, ymax))

    pl.plot(X, F, 'k--', lw=2)
    pl.axvline(x, color='r')
    pl.axvline(info[-1]['xbest'], color='g')
    pl.gca().set_xticklabels([])
    pl.gca().set_yticklabels([])
    pl.ylabel('posterior')

    pl.subplot(212)
    pl.cla()
    pl.plot(X, index(X), lw=2)
    pl.axvline(x, color='r')
    pl.axis(xmin=xmin, xmax=xmax)
    pl.gca().set_xticklabels([])
    pl.gca().set_yticklabels([])
    pl.ylabel('acquisition')
    pl.draw()

if __name__ == '__main__':
    T = 100
    sigma = 0.05
    gp = pygp.BasicGP(sigma, 1.0, 0.1, kernel='matern3')
    f = pybo.models.GPModel([3, 5], gp)

    pybo.policies.solve_bayesopt(f,
                                 f.bounds,
                                 sigma,
                                 policy='ucb',
                                 kernel=gp._kernel.copy(),
                                 callback=callback)
<|endoftext|>


<commit_before><commit_msg>FEM: solver base writer, add document as attribut<commit_after><|endoftext|>


<commit_before>import flask
from flask import render_template
from datetime import datetime, timedelta
from collections import defaultdict
import json

import sensors
from conf import read as read_conf
import logging
import database

app = flask.Flask(__name__)
conf = None

def db():
    db = getattr(flask.g, 'db', None)
    if db is None:
        logging.info('Connecting to DB.')
        flask.g.db = database.Database(conf['common']['database'])
    return flask.g.db

@app.teardown_appcontext
def close_db(error):
    logging.info('Closing DB.')

@app.route('/sensor/<sensor_id>')
def climon(sensor_id):
    temp, hum = sensors.get_by_id(conf, sensor_id)()
    return '%f %f' % (temp, hum)

range_dates = dict(
	day=lambda d: (d, d + timedelta(days=1)),
	week=lambda d: (d - timedelta(days=6), d - timedelta(days=1)),
	month=lambda d: (d - timedelta(days=30), d + timedelta(days=1)),
	all=lambda d: (db().getDateSpan()),
)

@app.route('/data/<range>/<yyyymmdd>')
def ganydata(range, yyyymmdd):
    assert range in range_dates
    day = datetime.strptime(yyyymmdd, '%Y%m%d')
    from_date, to_date = range_dates[range](day)
    sensor_data = {}
    for sensor_id in sensors.iter_ids(conf):
        sensor_data[sensor_id] = defaultdict(lambda: [])
        for d, avg_temp, avg_hum in db().get_stats(sensor_id, from_date, to_date, range):
            sensor_data[sensor_id]['temperatures'].append(dict(x=d.strftime('%Y%m%dT%H%M%S'), y=avg_temp))
            sensor_data[sensor_id]['humidities'].append(dict(x=d.strftime('%Y%m%dT%H%M%S'), y=avg_hum))
    return json.dumps(sensor_data)

@app.route('/')
def index():
    timestamp = datetime.now()
    sensor_confs=dict((sensor_id, conf['sensor:%s' % sensor_id]) for sensor_id in sensors.iter_ids(conf))
    return render_template('index.html', date=timestamp.strftime('%Y%m%d'), sensor_confs=sensor_confs)

def main(conf_fname):
    global conf

    logging.basicConfig(filename='climon.log',
            format='%(asctime)s %(levelname)s WEB %(message)s',
            level=logging.DEBUG)

    logging.info('Reading conf')
    conf = read_conf(conf_fname)
    logging.info('Reading conf done')
    app.run(debug=False, host='0.0.0.0', threaded=True, port=int(conf['common']['port']))

if __name__ == '__main__':
    import daemon
    import sys

    class Daemon(daemon.Daemon):

        def run(self):
            main('climon.conf')

    daemon = Daemon(pidfile='web.pid')

    if 'start' == sys.argv[1]:
        daemon.start()
    elif 'stop' == sys.argv[1]:
        daemon.stop()
    elif 'restart' == sys.argv[1]:
        daemon.restart()
<commit_msg>Fix week range.<commit_after>import flask
from flask import render_template
from datetime import datetime, timedelta
from collections import defaultdict
import json

import sensors
from conf import read as read_conf
import logging
import database

app = flask.Flask(__name__)
conf = None

def db():
    db = getattr(flask.g, 'db', None)
    if db is None:
        logging.info('Connecting to DB.')
        flask.g.db = database.Database(conf['common']['database'])
    return flask.g.db

@app.teardown_appcontext
def close_db(error):
    logging.info('Closing DB.')

@app.route('/sensor/<sensor_id>')
def climon(sensor_id):
    temp, hum = sensors.get_by_id(conf, sensor_id)()
    return '%f %f' % (temp, hum)

range_dates = dict(
	day=lambda d: (d, d + timedelta(days=1)),
	week=lambda d: (d - timedelta(days=6), d + timedelta(days=1)),
	month=lambda d: (d - timedelta(days=30), d + timedelta(days=1)),
	all=lambda d: (db().getDateSpan()),
)

@app.route('/data/<range>/<yyyymmdd>')
def ganydata(range, yyyymmdd):
    assert range in range_dates
    day = datetime.strptime(yyyymmdd, '%Y%m%d')
    from_date, to_date = range_dates[range](day)
    sensor_data = {}
    for sensor_id in sensors.iter_ids(conf):
        sensor_data[sensor_id] = defaultdict(lambda: [])
        for d, avg_temp, avg_hum in db().get_stats(sensor_id, from_date, to_date, range):
            sensor_data[sensor_id]['temperatures'].append(dict(x=d.strftime('%Y%m%dT%H%M%S'), y=avg_temp))
            sensor_data[sensor_id]['humidities'].append(dict(x=d.strftime('%Y%m%dT%H%M%S'), y=avg_hum))
    return json.dumps(sensor_data)

@app.route('/')
def index():
    timestamp = datetime.now()
    sensor_confs=dict((sensor_id, conf['sensor:%s' % sensor_id]) for sensor_id in sensors.iter_ids(conf))
    return render_template('index.html', date=timestamp.strftime('%Y%m%d'), sensor_confs=sensor_confs)

def main(conf_fname):
    global conf

    logging.basicConfig(filename='climon.log',
            format='%(asctime)s %(levelname)s WEB %(message)s',
            level=logging.DEBUG)

    logging.info('Reading conf')
    conf = read_conf(conf_fname)
    logging.info('Reading conf done')
    app.run(debug=False, host='0.0.0.0', threaded=True, port=int(conf['common']['port']))

if __name__ == '__main__':
    import daemon
    import sys

    class Daemon(daemon.Daemon):

        def run(self):
            main('climon.conf')

    daemon = Daemon(pidfile='web.pid')

    if 'start' == sys.argv[1]:
        daemon.start()
    elif 'stop' == sys.argv[1]:
        daemon.stop()
    elif 'restart' == sys.argv[1]:
        daemon.restart()
<|endoftext|>


<commit_before>from __future__ import division

from .mesh import MeshVisual
import numpy as np
from numpy.linalg import norm
from ..util.transforms import rotation_matrix
from ..color import ColorArray

class TubeVisual(MeshVisual):
    """Displays a tube around a piecewise-linear path.

    The tube mesh is corrected following its Frenet curvature and
    torsion such that it varies smoothly along the curve, including if
    the tube is closed.

    Parameters
    ----------
    points : ndarray
        An array of (x, y, z) points describing the path along which the
        tube will be extruded.
    radius : float
        The radius of the tube. Defaults to 1.0.
    color : Color | ColorArray
        The color(s) to use when drawing the tube. The same color is
        applied to each vertex of the mesh surrounding each point of
        the line. If the input is a ColorArray, the argument will be
        cycled; for instance if 'red' is passed then the entire tube
        will be red, or if ['green', 'blue'] is passed then the points
        will alternate between these colours. Defaults to 'purple'.
    tube_points : int
        The number of points in the circle-approximating polygon of the
        tube's cross section. Defaults to 8.
    shading : str | None
        Same as for the `MeshVisual` class. Defaults to 'smooth'.
    vertex_colors: ndarray | None
        Same as for the `MeshVisual` class.
    face_colors: ndarray | None
        Same as for the `MeshVisual` class.
    mode : str
        Same as for the `MeshVisual` class. Defaults to 'triangles'.

    """
    def __init__(self, points, radius=1.0, tube_points=8,
                 closed=False,
                 shading='smooth',
                 vertex_colors=None,
                 face_colors=None,
                 color='purple',
                 mode='triangles'):

        points = np.array(points)

        tangents, normals, binormals = _frenet_frames(points, closed)

        segments = len(points) - 1

        # get the positions of each vertex
        grid = np.zeros((len(points), tube_points, 3))
        for i in range(len(points)):
            pos = points[i]
            normal = normals[i]
            binormal = binormals[i]

            # Add a vertex for each point on the circle
            v = np.arange(tube_points,
                          dtype=np.float) / tube_points * 2 * np.pi
            cx = -1. * radius * np.cos(v)
            cy = radius * np.sin(v)
            grid[i] = (pos + cx[:, np.newaxis]*normal +
                       cy[:, np.newaxis]*binormal)

        # construct the mesh
        indices = []
        for i in range(segments):
            for j in range(tube_points):
                ip = (i+1) % segments if closed else i+1
                jp = (j+1) % tube_points

                index_a = i*tube_points + j
                index_b = ip*tube_points + j
                index_c = ip*tube_points + jp
                index_d = i*tube_points + jp

                indices.append([index_a, index_b, index_d])
                indices.append([index_b, index_c, index_d])

        vertices = grid.reshape(grid.shape[0]*grid.shape[1], 3)

        color = ColorArray(color)
        if vertex_colors is None:
            point_colors = np.resize(color.rgba,
                                     (len(points), 4))
            vertex_colors = np.repeat(point_colors, tube_points, axis=0)

        indices = np.array(indices, dtype=np.uint32)

        MeshVisual.__init__(self, vertices, indices,
                            vertex_colors=vertex_colors,
                            face_colors=face_colors,
                            # color=color,
                            shading=shading,
                            mode=mode)

    def draw(self, transforms):
        MeshVisual.draw(self, transforms)

def _frenet_frames(points, closed):
    '''Calculates and returns the tangents, normals and binormals for
    the tube.'''
    tangents = np.zeros((len(points), 3))
    normals = np.zeros((len(points), 3))

    epsilon = 0.0001

    # Compute tangent vectors for each segment
    tangents = np.roll(points, -1, axis=0) - np.roll(points, 1, axis=0)
    mags = np.sqrt(np.sum(tangents * tangents, axis=1))
    tangents /= mags[:, np.newaxis]

    # Get initial normal and binormal
    t = np.abs(tangents[0])

    smallest = np.argmin(t)
    normal = np.zeros(3)
    normal[smallest] = 1.

    vec = np.cross(tangents[0], normal)

    normals[0] = np.cross(tangents[0], vec)

    # Compute normal and binormal vectors along the path
    for i in range(1, len(points)):
        normals[i] = normals[i-1]

        vec = np.cross(tangents[i-1], tangents[i])
        if norm(vec) > epsilon:
            vec /= norm(vec)

            theta = np.arccos(np.clip(tangents[i-1].dot(tangents[i]), -1, 1))
            normals[i] = rotation_matrix(vec, theta).dot(normals[i])

    if closed:
        theta = np.arccos(np.clip(normals[0].dot(normals[-1]), -1, 1))
        theta /= len(points) - 1

        if tangents[0].dot(np.cross(normals[0], normals[-1])) > 0:
            theta *= -1.

        for i in range(1, len(points)):
            normals[i] = rotation_matrix(tangents[i], theta*i).dot(normals[i])

    binormals = np.cross(tangents, normals)

    return tangents, normals, binormals
<commit_msg>Removed debug comment<commit_after>from __future__ import division

from .mesh import MeshVisual
import numpy as np
from numpy.linalg import norm
from ..util.transforms import rotation_matrix
from ..color import ColorArray

class TubeVisual(MeshVisual):
    """Displays a tube around a piecewise-linear path.

    The tube mesh is corrected following its Frenet curvature and
    torsion such that it varies smoothly along the curve, including if
    the tube is closed.

    Parameters
    ----------
    points : ndarray
        An array of (x, y, z) points describing the path along which the
        tube will be extruded.
    radius : float
        The radius of the tube. Defaults to 1.0.
    color : Color | ColorArray
        The color(s) to use when drawing the tube. The same color is
        applied to each vertex of the mesh surrounding each point of
        the line. If the input is a ColorArray, the argument will be
        cycled; for instance if 'red' is passed then the entire tube
        will be red, or if ['green', 'blue'] is passed then the points
        will alternate between these colours. Defaults to 'purple'.
    tube_points : int
        The number of points in the circle-approximating polygon of the
        tube's cross section. Defaults to 8.
    shading : str | None
        Same as for the `MeshVisual` class. Defaults to 'smooth'.
    vertex_colors: ndarray | None
        Same as for the `MeshVisual` class.
    face_colors: ndarray | None
        Same as for the `MeshVisual` class.
    mode : str
        Same as for the `MeshVisual` class. Defaults to 'triangles'.

    """
    def __init__(self, points, radius=1.0, tube_points=8,
                 closed=False,
                 shading='smooth',
                 vertex_colors=None,
                 face_colors=None,
                 color='purple',
                 mode='triangles'):

        points = np.array(points)

        tangents, normals, binormals = _frenet_frames(points, closed)

        segments = len(points) - 1

        # get the positions of each vertex
        grid = np.zeros((len(points), tube_points, 3))
        for i in range(len(points)):
            pos = points[i]
            normal = normals[i]
            binormal = binormals[i]

            # Add a vertex for each point on the circle
            v = np.arange(tube_points,
                          dtype=np.float) / tube_points * 2 * np.pi
            cx = -1. * radius * np.cos(v)
            cy = radius * np.sin(v)
            grid[i] = (pos + cx[:, np.newaxis]*normal +
                       cy[:, np.newaxis]*binormal)

        # construct the mesh
        indices = []
        for i in range(segments):
            for j in range(tube_points):
                ip = (i+1) % segments if closed else i+1
                jp = (j+1) % tube_points

                index_a = i*tube_points + j
                index_b = ip*tube_points + j
                index_c = ip*tube_points + jp
                index_d = i*tube_points + jp

                indices.append([index_a, index_b, index_d])
                indices.append([index_b, index_c, index_d])

        vertices = grid.reshape(grid.shape[0]*grid.shape[1], 3)

        color = ColorArray(color)
        if vertex_colors is None:
            point_colors = np.resize(color.rgba,
                                     (len(points), 4))
            vertex_colors = np.repeat(point_colors, tube_points, axis=0)

        indices = np.array(indices, dtype=np.uint32)

        MeshVisual.__init__(self, vertices, indices,
                            vertex_colors=vertex_colors,
                            face_colors=face_colors,
                            shading=shading,
                            mode=mode)

    def draw(self, transforms):
        MeshVisual.draw(self, transforms)

def _frenet_frames(points, closed):
    '''Calculates and returns the tangents, normals and binormals for
    the tube.'''
    tangents = np.zeros((len(points), 3))
    normals = np.zeros((len(points), 3))

    epsilon = 0.0001

    # Compute tangent vectors for each segment
    tangents = np.roll(points, -1, axis=0) - np.roll(points, 1, axis=0)
    mags = np.sqrt(np.sum(tangents * tangents, axis=1))
    tangents /= mags[:, np.newaxis]

    # Get initial normal and binormal
    t = np.abs(tangents[0])

    smallest = np.argmin(t)
    normal = np.zeros(3)
    normal[smallest] = 1.

    vec = np.cross(tangents[0], normal)

    normals[0] = np.cross(tangents[0], vec)

    # Compute normal and binormal vectors along the path
    for i in range(1, len(points)):
        normals[i] = normals[i-1]

        vec = np.cross(tangents[i-1], tangents[i])
        if norm(vec) > epsilon:
            vec /= norm(vec)

            theta = np.arccos(np.clip(tangents[i-1].dot(tangents[i]), -1, 1))
            normals[i] = rotation_matrix(vec, theta).dot(normals[i])

    if closed:
        theta = np.arccos(np.clip(normals[0].dot(normals[-1]), -1, 1))
        theta /= len(points) - 1

        if tangents[0].dot(np.cross(normals[0], normals[-1])) > 0:
            theta *= -1.

        for i in range(1, len(points)):
            normals[i] = rotation_matrix(tangents[i], theta*i).dot(normals[i])

    binormals = np.cross(tangents, normals)

    return tangents, normals, binormals
<|endoftext|>


<commit_before>''' Python 2.7.3
AUTHOR: Hunter Hammond
VERSION: 2.0
DEPENDENCIES: lxml
'''

import argparse
import os
import sys
import urllib2
import itertools
from collections import OrderedDict

import lxml.html as lh

class WolfSearch:
    def __init__(self, url_args):
        self.url_args = self.ProcessArgs(url_args)
        self.api_key = self.GetApiKey()
        self.html = self.GetWolfHTML(self.url_args)

    def ProcessArgs(self, url_args):
        clean_args = []
        if url_args:
            for arg in url_args:
                try:
                    if " " in arg:
                        clean_args = arg.split(" ")
                    else:
                        clean_args.append(arg)
                except IndexError:
                    sys.exit()
        return clean_args

    def GetApiKey(self):
        # Get API key from config.txt
        cfg_path = os.path.dirname(os.path.abspath(__file__))
        f = open(cfg_path + '/config.txt', 'r')
        api_key = f.readline()
        f.close()
        return api_key

    def GetWolfHTML(self, url_args):
        # Add clean_args and api_key to base_url
        base_url = 'http://api.wolframalpha.com/v2/query?input='
        try:
            clean_args = '+'.join(url_args)
        except AttributeError:
            sys.stderr.write("Argument list error! Expected list, got " + type(clean_args) + "\n")
            sys.exit()
        url = base_url + clean_args + '&appid=' + self.api_key

        # Get HTML response
        try:
            request = urllib2.Request(url, headers={ 'User-Agent': 'Mozilla/5.0' })
            return lh.parse(urllib2.urlopen(request))
        except urllib2.URLError:
            print 'WolfFail'
            sys.stderr.write('Failed to retrieve webpage.\n')
            sys.exit()

    def Search(self):
        # Parse webpage response
        titles = list(OrderedDict.fromkeys(self.html.xpath("//pod[@title != '' and @title != 'Number line' and @title != 'Input' and @title != 'Visual representation']/@title")))
        entries = []
        if titles:
            for title in titles:
                entry_xpath = "//pod[@title='" + title + "']/subpod/plaintext/text()"
                entry = self.html.xpath(entry_xpath)
                if entry:
                    entries.append(entry[0])

            entries = list(OrderedDict.fromkeys(entries))
            output_list = []
            for title, entry in itertools.izip(titles, entries):
                try:
                    output_list.append(title + ': ' + entry).encode('ascii', 'ignore')
                except (AttributeError, UnicodeEncodeError):
                    pass
            sys.stderr.write('\n'.join(output_list).encode('ascii', 'ignore'))
        else:
            print 'WolfNoMatch'

if __name__ == "__main__":
    # parse arguments
    parser = argparse.ArgumentParser()
    parser.add_argument("-s", "--search", help="get bing search results",
    action="store_true")
    parser.add_argument("-f", "--first", help="opens first link result",
    action="store_true")
    parser.add_argument("-o", "--openurl", help="open link directly",
    action="store_true")
    parser.add_argument("-w", "--wolfram", help="get wolfram search result",
    action="store_true")
    parser.add_argument("-i", "--incognito", help="open browser in incognito",
    action="store_true")
    parser.add_argument("URL_ARGS", nargs='*', help="Search keywords")
    args = parser.parse_args()
    wolf_search = WolfSearch(args.URL_ARGS)
    wolf_search.Search()

<commit_msg>cleaned up output & improved filtering of results<commit_after>''' Python 2.7.3
AUTHOR: Hunter Hammond
VERSION: 2.0
DEPENDENCIES: lxml
'''

import argparse
import os
import sys
import urllib2
import itertools
from collections import OrderedDict

import lxml.html as lh

class WolfSearch:
    def __init__(self, url_args):
        self.url_args = self.ProcessArgs(url_args)
        self.api_key = self.GetApiKey()
        self.html = self.GetWolfHTML(self.url_args)

    def ProcessArgs(self, url_args):
        clean_args = []
        if url_args:
            for arg in url_args:
                try:
                    if " " in arg:
                        clean_args = arg.split(" ")
                    else:
                        clean_args.append(arg)
                except IndexError:
                    sys.exit()
        return clean_args

    def GetApiKey(self):
        # Get API key from config.txt
        cfg_path = os.path.dirname(os.path.abspath(__file__))
        f = open(cfg_path + '/config.txt', 'r')
        api_key = f.readline()
        f.close()
        return api_key

    def GetWolfHTML(self, url_args):
        # Add clean_args and api_key to base_url
        base_url = 'http://api.wolframalpha.com/v2/query?input='
        try:
            clean_args = '+'.join(url_args)
        except AttributeError:
            sys.stderr.write("Argument list error! Expected list, got " + type(clean_args) + "\n")
            sys.exit()
        url = base_url + clean_args + '&appid=' + self.api_key

        # Get HTML response
        try:
            request = urllib2.Request(url, headers={ 'User-Agent': 'Mozilla/5.0' })
            return lh.parse(urllib2.urlopen(request))
        except urllib2.URLError:
            print 'WolfFail'
            sys.stderr.write('Failed to retrieve webpage.\n')
            sys.exit()

    def Search(self):
        # Parse webpage response
        titles = list(OrderedDict.fromkeys(self.html.xpath("//pod[@title != '' and @title != 'Number line' and @title != 'Input' and @title != 'Visual representation' and @title != 'Input interpretation']/@title")))
        entries = []
        if titles:
            for title in titles:
                entry_xpath = "//pod[@title='" + title + "']/subpod/plaintext/text()"
                entry = self.html.xpath(entry_xpath)
                if entry:
                    entries.append(entry[0])
            entries = list(OrderedDict.fromkeys(entries))
            output_list = []
            if len(entries) == 1 and entries[0] == '{}':
                print 'WolfNoMatch'
                sys.exit()
            for title, entry in itertools.izip(titles, entries):
                try:
                    if ' |' in entry:
                        entry = '\n\t' + entry.replace(' |', ':').replace('\n', '\n\t')
                    if title == 'Result':
                        output_list.append(entry).encode('ascii', 'ignore')
                    else:
                        output_list.append(title + ': ' + entry).encode('ascii', 'ignore')
                except (AttributeError, UnicodeEncodeError):
                    pass
            sys.stderr.write('\n'.join(output_list).encode('ascii', 'ignore'))
        else:
            print 'WolfNoMatch'

if __name__ == "__main__":
    # parse arguments
    parser = argparse.ArgumentParser()
    parser.add_argument("-s", "--search", help="get bing search results",
    action="store_true")
    parser.add_argument("-f", "--first", help="opens first link result",
    action="store_true")
    parser.add_argument("-o", "--openurl", help="open link directly",
    action="store_true")
    parser.add_argument("-w", "--wolfram", help="get wolfram search result",
    action="store_true")
    parser.add_argument("-i", "--incognito", help="open browser in incognito",
    action="store_true")
    parser.add_argument("URL_ARGS", nargs='*', help="Search keywords")
    args = parser.parse_args()
    wolf_search = WolfSearch(args.URL_ARGS)
    wolf_search.Search()

<|endoftext|>


<commit_before>#!/usr/bin/env python2
# projectS and projectC were written by Gabriele Farina.

import time

start = time.time()

import argparse
import cv2
import os
import dlib

import numpy as np
np.set_printoptions(precision=2)
import openface

from matplotlib import cm

fileDir = os.path.dirname(os.path.realpath(__file__))
modelDir = os.path.join(fileDir, '..', 'models')
dlibModelDir = os.path.join(modelDir, 'dlib')
openfaceModelDir = os.path.join(modelDir, 'openface')

def getRep(bgrImg):
    start = time.time()
    if bgrImg is None:
        raise Exception("Unable to load image/frame")

    rgbImg = cv2.cvtColor(bgrImg, cv2.COLOR_BGR2RGB)

    if args.verbose:
        print("  + Original size: {}".format(rgbImg.shape))
    if args.verbose:
        print("Loading the image took {} seconds.".format(time.time() - start))

    start = time.time()

    # Get all bounding boxes
    bb = align.getAllFaceBoundingBoxes(rgbImg)

    if bb is None:
        # raise Exception("Unable to find a face: {}".format(imgPath))
        return None
    if args.verbose:
        print("Face detection took {} seconds.".format(time.time() - start))

    start = time.time()

    alignedFaces = []
    for box in bb:
        alignedFaces.append(
            align.align(
                args.imgDim,
                rgbImg,
                box,
                landmarkIndices=openface.AlignDlib.OUTER_EYES_AND_NOSE))

    if alignedFaces is None:
        raise Exception("Unable to align the frame")
    if args.verbose:
        print("Alignment took {} seconds.".format(time.time() - start))

    start = time.time()

    reps = []
    for alignedFace in alignedFaces:
        reps.append(net.forward(alignedFace))

    if args.verbose:
        print("Neural network forward pass took {} seconds.".format(
            time.time() - start))

    return reps

def projectS(rho, theta, z):
    p = np.array([np.sqrt(3.) * rho * (np.cos(theta) + np.sin(theta)) / 2.,
                  z + 1. + rho * (np.cos(theta) - np.sin(theta)) / 2.])
    p += np.array([1.5, 0.5])
    p /= 3.
    return p

def projectC(x, y, z):
    rho = np.sqrt(x**2 + y**2)
    if x == 0 and y == 0:
        theta = 0
    elif x >= 0:
        theta = np.arcsin(y / rho)
    else:
        theta = -np.arcsin(y / rho) + np.pi

    return projectS(rho, theta, z)

def draw(pts=[], clrs=[], cSz=400):
    def toFrame(x):
        return tuple((cSz * x).astype(np.int32))

    cFrame = np.full((cSz, cSz, 3), 255, dtype=np.uint8)

    for z in np.linspace(-1, 1, 9):
        r = np.sqrt(1. - z**2)
        last = None
        for theta in np.linspace(0, 2 * np.pi, 50):
            x = toFrame(projectS(r, theta, z))
            if last is not None:
                cv2.line(cFrame, x, last, color=(0, 0, 0))
            last = x

    for x in np.linspace(-1, 1, 9):
        last = None
        for theta in np.linspace(0, 2 * np.pi, 50):
            r = np.sqrt(1. - x**2)
            z = r * np.sin(theta)
            y = r * np.cos(theta)
            # x = toFrame(projectS(r, theta, z))
            p = toFrame(projectC(x, y, z))
            if last is not None:
                cv2.line(cFrame, p, last, color=(0, 0, 0))
            last = p

    s = 1
    x = toFrame(projectC(-s, 0, 0))
    y = toFrame(projectC(s, 0, 0))
    cv2.line(cFrame, x, y, color=(0, 0, 0), thickness=4)

    x = toFrame(projectC(0, -s, 0))
    y = toFrame(projectC(0, s, 0))
    cv2.line(cFrame, x, y, color=(0, 0, 0), thickness=4)

    x = toFrame(projectC(0, 0, -s))
    y = toFrame(projectC(0, 0, s))
    cv2.line(cFrame, x, y, color=(0, 0, 0), thickness=4)

    for pt, c in zip(pts, clrs):
        fPt = toFrame(projectC(pt[0], pt[1], pt[2]))
        fPt_noz = toFrame(projectC(pt[0], pt[1], 0))
        fPt_nozy = toFrame(projectC(pt[0], 0, 0))
        fPt_nozx = toFrame(projectC(0, pt[1], 0))
        cv2.line(cFrame, fPt, fPt_noz, color=c, thickness=2)
        cv2.line(cFrame, fPt_noz, fPt_nozy, color=c, thickness=2)
        cv2.line(cFrame, fPt_noz, fPt_nozx, color=c, thickness=2)
        cv2.circle(cFrame, fPt, 5, color=c, thickness=-1)

    return cFrame

if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument(
        '--dlibFacePredictor',
        type=str,
        help="Path to dlib's face predictor.",
        default=os.path.join(
            dlibModelDir,
            "shape_predictor_68_face_landmarks.dat"))
    parser.add_argument(
        '--networkModel',
        type=str,
        help="Path to Torch network model.",
        default='nn4.small2.3d.v1.t7')
    # Download the 3D model from:
    # http://openface-models.storage.cmusatyalab.org/nn4.small2.3d.v1.t7
    parser.add_argument('--imgDim', type=int,
                        help="Default image dimension.", default=96)
    parser.add_argument(
        '--captureDevice',
        type=int,
        default=0,
        help='Capture device. 0 for latop webcam and 1 for usb webcam')
    # parser.add_argument('--width', type=int, default=640)
    # parser.add_argument('--height', type=int, default=480)
    parser.add_argument('--width', type=int, default=1280)
    parser.add_argument('--height', type=int, default=800)
    parser.add_argument('--scale', type=int, default=0.25)
    parser.add_argument('--threshold', type=float, default=0.5)
    parser.add_argument('--cuda', action='store_true')
    parser.add_argument('--verbose', action='store_true')

    args = parser.parse_args()

    align = openface.AlignDlib(args.dlibFacePredictor)
    net = openface.TorchNeuralNet(
        args.networkModel,
        imgDim=args.imgDim,
        cuda=args.cuda)

    # Capture device. Usually 0 will be webcam and 1 will be usb cam.
    video_capture = cv2.VideoCapture(args.captureDevice)
    video_capture.set(3, args.width)
    video_capture.set(4, args.height)

    cv2.namedWindow('video', cv2.WINDOW_NORMAL)

    class Tracker:

        def __init__(self, img, bb, rep):
            self.t = dlib.correlation_tracker()
            self.t.start_track(img, bb)
            self.rep = rep
            self.bb = bb
            self.pings = 0

        def updateRep(self, rep):
            self.pings = 0
            alpha = 0.9
            self.rep = alpha * self.rep + (1. - alpha) * rep
            return self.rep

        def overlap(self, bb):
            p = float(self.bb.intersect(bb).area()) / float(self.bb.area())
            return p > 0.3

        def ping(self):
            self.pings += 1

    trackers = []

    while True:
        ret, frame = video_capture.read()
        frame = cv2.flip(frame, 1)
        frameSmall = cv2.resize(frame, (int(args.width * args.scale),
                                        int(args.height * args.scale)))

        bbs = align.getAllFaceBoundingBoxes(frameSmall)

        pts, clrs = [], []
        for i, bb in enumerate(bbs):
            alignedFace = align.align(96, frameSmall, bb,
                                      landmarkIndices=openface.AlignDlib.INNER_EYES_AND_BOTTOM_LIP)
            rep = net.forward(alignedFace)

            center = bb.center()
            centerI = 0.7 * center.x * center.y / \
                (args.scale * args.scale * args.width * args.height)
            color_np = cm.Set1(centerI)
            color_cv = list(np.multiply(color_np[:3], 255))

            bl = (int(bb.left() / args.scale), int(bb.bottom() / args.scale))
            tr = (int(bb.right() / args.scale), int(bb.top() / args.scale))
            cv2.rectangle(frame, bl, tr, color=color_cv, thickness=3)

            tracked = False
            for i in xrange(len(trackers) - 1, -1, -1):
                t = trackers[i]
                t.t.update(frame)
                if t.overlap(bb):
                    rep = t.updateRep(rep)
                    pts.append(rep)
                    clrs.append(color_cv)
                    tracked = True
                    break

            if not tracked:
                trackers.append(Tracker(frame, bb, rep))
                pts.append(rep)
                clrs.append(color_cv)

        for i in xrange(len(trackers) - 1, -1, -1):
            t = trackers[i]
            t.ping()
            if t.pings > 10:
                del trackers[i]
                continue

            for j in range(i):
                if t.t.get_position().intersect(trackers[j].t.get_position()).area() / \
                   t.t.get_position().area() > 0.4:
                    del trackers[i]
                    continue

        cSz = 450
        sphere = np.copy(frame)
        sphere[0:cSz, 0:cSz, :] = draw(pts, clrs, cSz)
        alpha = 0.25
        beta = 1. - alpha
        cv2.putText(sphere, "CMU OpenFace", (50, 30),
                    cv2.FONT_HERSHEY_COMPLEX_SMALL, 2.,
                    (0, 0, 0), 1, cv2.cv.CV_AA)
        cv2.addWeighted(frame, alpha, sphere, beta, 0.0, frame)
        cv2.imshow('video', frame)

        if cv2.waitKey(1) & 0xFF == ord('q'):
            break

    video_capture.release()
    cv2.destroyAllWindows()
<commit_msg>Remove unused code.<commit_after>#!/usr/bin/env python2
# projectS and projectC were written by Gabriele Farina.

import time

start = time.time()

import argparse
import cv2
import os
import dlib

import numpy as np
np.set_printoptions(precision=2)
import openface

from matplotlib import cm

fileDir = os.path.dirname(os.path.realpath(__file__))
modelDir = os.path.join(fileDir, '..', 'models')
dlibModelDir = os.path.join(modelDir, 'dlib')

def getRep(bgrImg):
    start = time.time()
    if bgrImg is None:
        raise Exception("Unable to load image/frame")

    rgbImg = cv2.cvtColor(bgrImg, cv2.COLOR_BGR2RGB)

    if args.verbose:
        print("  + Original size: {}".format(rgbImg.shape))
    if args.verbose:
        print("Loading the image took {} seconds.".format(time.time() - start))

    start = time.time()

    # Get all bounding boxes
    bb = align.getAllFaceBoundingBoxes(rgbImg)

    if bb is None:
        # raise Exception("Unable to find a face: {}".format(imgPath))
        return None
    if args.verbose:
        print("Face detection took {} seconds.".format(time.time() - start))

    start = time.time()

    alignedFaces = []
    for box in bb:
        alignedFaces.append(
            align.align(
                args.imgDim,
                rgbImg,
                box,
                landmarkIndices=openface.AlignDlib.OUTER_EYES_AND_NOSE))

    if alignedFaces is None:
        raise Exception("Unable to align the frame")
    if args.verbose:
        print("Alignment took {} seconds.".format(time.time() - start))

    start = time.time()

    reps = []
    for alignedFace in alignedFaces:
        reps.append(net.forward(alignedFace))

    if args.verbose:
        print("Neural network forward pass took {} seconds.".format(
            time.time() - start))

    return reps

def projectS(rho, theta, z):
    p = np.array([np.sqrt(3.) * rho * (np.cos(theta) + np.sin(theta)) / 2.,
                  z + 1. + rho * (np.cos(theta) - np.sin(theta)) / 2.])
    p += np.array([1.5, 0.5])
    p /= 3.
    return p

def projectC(x, y, z):
    rho = np.sqrt(x**2 + y**2)
    if x == 0 and y == 0:
        theta = 0
    elif x >= 0:
        theta = np.arcsin(y / rho)
    else:
        theta = -np.arcsin(y / rho) + np.pi

    return projectS(rho, theta, z)

def draw(pts=[], clrs=[], cSz=400):
    def toFrame(x):
        return tuple((cSz * x).astype(np.int32))

    cFrame = np.full((cSz, cSz, 3), 255, dtype=np.uint8)

    for z in np.linspace(-1, 1, 9):
        r = np.sqrt(1. - z**2)
        last = None
        for theta in np.linspace(0, 2 * np.pi, 50):
            x = toFrame(projectS(r, theta, z))
            if last is not None:
                cv2.line(cFrame, x, last, color=(0, 0, 0))
            last = x

    for x in np.linspace(-1, 1, 9):
        last = None
        for theta in np.linspace(0, 2 * np.pi, 50):
            r = np.sqrt(1. - x**2)
            z = r * np.sin(theta)
            y = r * np.cos(theta)
            # x = toFrame(projectS(r, theta, z))
            p = toFrame(projectC(x, y, z))
            if last is not None:
                cv2.line(cFrame, p, last, color=(0, 0, 0))
            last = p

    s = 1
    x = toFrame(projectC(-s, 0, 0))
    y = toFrame(projectC(s, 0, 0))
    cv2.line(cFrame, x, y, color=(0, 0, 0), thickness=4)

    x = toFrame(projectC(0, -s, 0))
    y = toFrame(projectC(0, s, 0))
    cv2.line(cFrame, x, y, color=(0, 0, 0), thickness=4)

    x = toFrame(projectC(0, 0, -s))
    y = toFrame(projectC(0, 0, s))
    cv2.line(cFrame, x, y, color=(0, 0, 0), thickness=4)

    for pt, c in zip(pts, clrs):
        fPt = toFrame(projectC(pt[0], pt[1], pt[2]))
        fPt_noz = toFrame(projectC(pt[0], pt[1], 0))
        fPt_nozy = toFrame(projectC(pt[0], 0, 0))
        fPt_nozx = toFrame(projectC(0, pt[1], 0))
        cv2.line(cFrame, fPt, fPt_noz, color=c, thickness=2)
        cv2.line(cFrame, fPt_noz, fPt_nozy, color=c, thickness=2)
        cv2.line(cFrame, fPt_noz, fPt_nozx, color=c, thickness=2)
        cv2.circle(cFrame, fPt, 5, color=c, thickness=-1)

    return cFrame

if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument(
        '--dlibFacePredictor',
        type=str,
        help="Path to dlib's face predictor.",
        default=os.path.join(
            dlibModelDir,
            "shape_predictor_68_face_landmarks.dat"))
    parser.add_argument(
        '--networkModel',
        type=str,
        help="Path to Torch network model.",
        default='nn4.small2.3d.v1.t7')
    # Download the 3D model from:
    # http://openface-models.storage.cmusatyalab.org/nn4.small2.3d.v1.t7
    parser.add_argument('--imgDim', type=int,
                        help="Default image dimension.", default=96)
    parser.add_argument(
        '--captureDevice',
        type=int,
        default=0,
        help='Capture device. 0 for latop webcam and 1 for usb webcam')
    # parser.add_argument('--width', type=int, default=640)
    # parser.add_argument('--height', type=int, default=480)
    parser.add_argument('--width', type=int, default=1280)
    parser.add_argument('--height', type=int, default=800)
    parser.add_argument('--scale', type=int, default=0.25)
    parser.add_argument('--threshold', type=float, default=0.5)
    parser.add_argument('--cuda', action='store_true')
    parser.add_argument('--verbose', action='store_true')

    args = parser.parse_args()

    align = openface.AlignDlib(args.dlibFacePredictor)
    net = openface.TorchNeuralNet(
        args.networkModel,
        imgDim=args.imgDim,
        cuda=args.cuda)

    # Capture device. Usually 0 will be webcam and 1 will be usb cam.
    video_capture = cv2.VideoCapture(args.captureDevice)
    video_capture.set(3, args.width)
    video_capture.set(4, args.height)

    cv2.namedWindow('video', cv2.WINDOW_NORMAL)

    class Tracker:

        def __init__(self, img, bb, rep):
            self.t = dlib.correlation_tracker()
            self.t.start_track(img, bb)
            self.rep = rep
            self.bb = bb
            self.pings = 0

        def updateRep(self, rep):
            self.pings = 0
            alpha = 0.9
            self.rep = alpha * self.rep + (1. - alpha) * rep
            return self.rep

        def overlap(self, bb):
            p = float(self.bb.intersect(bb).area()) / float(self.bb.area())
            return p > 0.3

        def ping(self):
            self.pings += 1

    trackers = []

    while True:
        ret, frame = video_capture.read()
        frame = cv2.flip(frame, 1)
        frameSmall = cv2.resize(frame, (int(args.width * args.scale),
                                        int(args.height * args.scale)))

        bbs = align.getAllFaceBoundingBoxes(frameSmall)

        pts, clrs = [], []
        for i, bb in enumerate(bbs):
            alignedFace = align.align(96, frameSmall, bb,
                                      landmarkIndices=openface.AlignDlib.INNER_EYES_AND_BOTTOM_LIP)
            rep = net.forward(alignedFace)

            center = bb.center()
            centerI = 0.7 * center.x * center.y / \
                (args.scale * args.scale * args.width * args.height)
            color_np = cm.Set1(centerI)
            color_cv = list(np.multiply(color_np[:3], 255))

            bl = (int(bb.left() / args.scale), int(bb.bottom() / args.scale))
            tr = (int(bb.right() / args.scale), int(bb.top() / args.scale))
            cv2.rectangle(frame, bl, tr, color=color_cv, thickness=3)

            tracked = False
            for i in xrange(len(trackers) - 1, -1, -1):
                t = trackers[i]
                t.t.update(frame)
                if t.overlap(bb):
                    rep = t.updateRep(rep)
                    pts.append(rep)
                    clrs.append(color_cv)
                    tracked = True
                    break

            if not tracked:
                trackers.append(Tracker(frame, bb, rep))
                pts.append(rep)
                clrs.append(color_cv)

        for i in xrange(len(trackers) - 1, -1, -1):
            t = trackers[i]
            t.ping()
            if t.pings > 10:
                del trackers[i]
                continue

            for j in range(i):
                if t.t.get_position().intersect(trackers[j].t.get_position()).area() / \
                   t.t.get_position().area() > 0.4:
                    del trackers[i]
                    continue

        cSz = 450
        sphere = np.copy(frame)
        sphere[0:cSz, 0:cSz, :] = draw(pts, clrs, cSz)
        alpha = 0.25
        beta = 1. - alpha
        cv2.putText(sphere, "CMU OpenFace", (50, 30),
                    cv2.FONT_HERSHEY_COMPLEX_SMALL, 2.,
                    (0, 0, 0), 1, cv2.cv.CV_AA)
        cv2.addWeighted(frame, alpha, sphere, beta, 0.0, frame)
        cv2.imshow('video', frame)

        if cv2.waitKey(1) & 0xFF == ord('q'):
            break

    video_capture.release()
    cv2.destroyAllWindows()
<|endoftext|>


<commit_before>#!/usr/bin/python
# -*- coding: utf-8 -*-
import os, json
from stravalib import client, unithelper
import pandas as pd
from datetime import datetime, timedelta
from sqlalchemy import create_engine
from unidecode import unidecode

def coerce_uni(string):
    ''' coerce ascii2 unicode for a string'''
    if string:
        string = unicode(unidecode(string))
    return string

def GetActivities(client, startDate, endDate, limit):
    # Returns a list of Strava activity objects, up to the number specified by
    # limit
    startDate = datetime.strptime(startDate, '%Y-%m-%d')
    endDate = datetime.strptime(endDate, '%Y-%m-%d')
    print "limit is: " + str(limit)
    activities = client.get_activities(before=endDate,
                                       after=startDate,
                                       limit=limit)
    return activities

def GetStreams(client, activity, types, resolution):
    # Returns a Strava 'stream', which is timeseries data from an activity
    streams = client.get_activity_streams(activity,
                                          types=types,
                                          series_type='distance',
                                          resolution=resolution)
    return streams

def DataFrame(dict, types):
    # Converts a Stream into a dataframe, and returns the dataframe
    df = pd.DataFrame()
    for item in types:
        if item in dict.keys():
            df.append(item.data)
    df.fillna('', inplace=True)
    return df

def ParseActivity(client, act, types, resolution):
    # Parses an activity from an athlete into a dataframe.  Returns the
    # dataframe
    act_id = act.id
    name = coerce_uni(act.name)
    if act.trainer==False and act.manual==False:
        streams = GetStreams(client, act_id, types, resolution)
        df = pd.DataFrame()
        # Write each row to a dataframe
        for item in types:
            if item in streams.keys():
                df[item] = pd.Series(streams[item].data, index=None)
            df['act_id'] = act.id
            df['act_startDate'] = pd.to_datetime(act.start_date_local)
    else:
       return
    return df

def get_acts_in_db(engine, table_name):
    # Return a list of already cached activities in the database
    already_dl_act_id_list = []
    try:
        args = 'SELECT Distinct act_id from "%s"' % (table_name)
        df = pd.read_sql(args, engine)
        already_dl_act_id_list = df['act_id'].tolist()
    except:
        print "no activities in database!  downloading all activities in range..."

    return already_dl_act_id_list

def check_if_new_act(act_list, already_dl_act_id_list):
    # Returns new activities not yet in the database
    for act in list(act_list):
        if act.id in already_dl_act_id_list:
            act_list.remove(act)

    return act_list

def loop_activities(client, activities, already_dl_act_id_list, types):
    # Loops through a list of activities, creating a dict of dataframes, one
    # entry for each activity
    df_lst = {}
    for act in activities:
        if act.id not in already_dl_act_id_list and act.trainer==False and act.manual==False:
            try:
                df_lst[act.start_date] = ParseActivity(client, act, types)
            except:
                print 'error adding to list, moving on!'
    return df_lst

def calctime(time_sec, startdate):
    # calculates timestamp from the elapsed time.  Returns a datetime value.
    timestamp = startdate + timedelta(seconds=float(time_sec))

    return timestamp

def split_lat(series):
    # splits a list and retunrs the 0th item.
    lat = series[0]
    return lat

def split_long(series):
    # splits a list and retunrs the 1th item.
    long = series[1]
    return long

def create_points(lat_series, long_series):
    # Creates a string from a lat/long column to map to a Geography Point
    # datatype in PostGIS
    point_col = 'Point(' + str(long_series) + ' ' + str(lat_series) + ')'

    return point_col

def concatdf(df_lst):
    # concatenate a dictionary of dataframes into one dataframe.  Returns the
    # concat'ed dataframe
    if df_lst:
        try:
            df = pd.concat(df_lst,
                           ignore_index=False,
                           copy=False)
        except:
            print "error concatenating dataframes!"
    else:
        df = pd.DataFrame()

    return df

def cleandf(df_total):
    # Applies a number of functions to clean out a dataframe for final
    # processing
    df_total.fillna(0, inplace=True)
    df_total.reset_index(level=0, inplace=True)
    df_total['timestamp'] = map(
        calctime, df_total['time'], df_total['act_startDate'])
    df_total['timestamp'].astype('datetime64[ns]')
    df_total['lat'] = map(split_lat, (df_total['latlng']))
    df_total['long'] = map(split_long, (df_total['latlng']))
    df_total.drop(['latlng', 'act_startDate', 'index'], axis=1, inplace=True)
    df_total['point'] = map(create_points, df_total['lat'], df_total['long'])
    df_total.rename(columns={'time': 'elapsed_time',
                             'distance': 'elapsed_dist'},
                    inplace=True)

    return df_total

def process_activities(client, limit, types, engine, table_name):
    # Applies a number of functions to get and convert and clean
    activities = GetActivities(client, limit)
    already_dl_act_id_list = get_acts_in_db(engine, table_name)
    df_total = loop_activities(
        client, activities, already_dl_act_id_list, types)
    df_total = concatdf(df_total)
    if not df_total.empty:
        df_total = cleandf(df_total)
    else:
        print "no new data to clean"

    return df_total

def df_to_csv(df, dst_file):
    # Writes a dataframe to a csv file. Returns none
    print "writing data to csv..."
    df.to_csv(dst_file, chunksize=100)
    # except:
    #    print "error writing file to CSV!"

    return None

def zip_and_delete(src_file, dst_file):
    # Zips up and deletes a file.  Returns none.
    print "zipping and deleting file..."
    try:
        with zipfile.ZipFile(dst_file, 'w', zipfile.ZIP_DEFLATED) as myzip:
            myzip.write(src_file)
            os.remove(src_file)
    except:
        print "error zipping file!"
    return None

def fully_process_acts(client, limit, types, csv_file, zip_file, engine, table_name):
    df = process_activities(client, limit, types, engine, table_name)
    no_new_data = False
    if df.empty:
        no_new_data = True
        args = 'SELECT * from "%s"' % (table_name)
        df = pd.read_sql(args, engine)

    df_to_csv(df, csv_file)
    zip_and_delete(csv_file, zip_file)

    return df, df.ix[[0, 1, 2, 3, 4, 5]], no_new_data

def drop_table(engine, table_name):
    try:
        print "dropping table and views... " + table_name
        engine.execute('DROP TABLE "%s" CASCADE; COMMIT;' % (table_name))
        print "table dropped successfully!"
        engine.execute('DROP INDEX IF EXISTS id_geog_actid CASCADE; COMMIT;')
        print "index dropped successfully!"
    except:
        print "error dropping table! " + table_name

def df_to_postgis_db(df, engine, table_name, no_new_data):
    # Write a dataframe to a PostGIS database.  Returns none.
    if no_new_data == False:

        try:
            engine.execute("CREATE EXTENSION postgis;")
        except:
            print "postgis extension already instealled!"

        # create table for the data
        meta = MetaData(engine)
        my_table = Table(table_name, meta,
                         Column('id', Integer),
                         Column('act_startDate', DateTime),
                         Column('timestamp', DateTime),
                         Column('time', Float),
                         Column('act_id', Float),
                         Column('act_name', VARCHAR(100)),
                         Column('distance', Float),
                         Column('altitude', Float),
                         Column('velocity_smooth', Float),
                         Column('grade_smooth', Float),
                         Column('lat', Float),
                         Column('long', Float),
                         )

        Index('id_geog_actid',
              my_table.c.id,
              my_table.c.act_id)

        try:
            print "Creating table..."
            my_table.create(engine)
            print "adding point column..."
            engine.execute(
                'ALTER TABLE "%s" ADD "point" geography(POINT,4326);' % (table_name))
            print "created table " + table_name
        except:
            print "table already exists!  Appending existing table with more data..."

        print "Write data to database..."
        df.to_sql(table_name, engine,
                  if_exists='append',
                  chunksize=500,
                  index=True,
                  index_label='id')
        print "inserted data into table!"

        print "creating view..."
        my_view = 'SELECT ath_id, act_id, act_name, ST_MakeLine(st_point(long, lat) ORDER BY id) AS linestring \
                   FROM "%s" GROUP BY act_id, act_name;' % (table_name)

        try:
            engine.execute('CREATE OR REPLACE VIEW V_%s_LS AS ' %
                           (table_name) + my_view)
        except:
            print "error creating view! - HEADS UP FOR PROBLEMS!"

    return 'v_%s_ls'.lower() % (table_name)

def to_geojson_file(engine, view_name, filename):
    # check the output, this geojson isn't perfect as expected...
    geojson_sql = "SELECT row_to_json(fc) \
                   FROM (SELECT 'FeatureCollection' As type, \
                        array_to_json(array_agg(f)) As features \
                        FROM (SELECT 'Feature' As type, \
                                ST_AsGeoJSON(linestring)::json As geometry, \
                                row_to_json((SELECT l \
                                    FROM (SELECT act_id, act_name) As l)) \
                                As properties \
                            FROM %s As lg) \
                        As f ) \
                    As fc;" % (view_name)

    result = engine.execute(geojson_sql)
    for row in result:
        data = row.values()

    geojson_data = str(json.dumps(data))
    #geojson_data = [dict(row) for row in result]

    with open(filename, 'w') as f:
        try:
            f.writelines(str(geojson_data)[1:-1])
        except:
            "error exporting geojson!  Try again later :("

    return None

def to_geojson_data(engine, view_name, ath_id):
    # check the output, this geojson isn't perfect as expected...
    geojson_sql = "SELECT row_to_json(fc) \
                   FROM (SELECT 'FeatureCollection' As type, \
                        array_to_json(array_agg(f)) As features \
                        FROM (SELECT 'Feature' As type, \
                                ST_AsGeoJSON(linestring,6)::json As geometry, \
                                row_to_json((SELECT l \
                                    FROM (SELECT act_id as id, act_name as na, act_type as ty) As l)) \
                                As properties \
                            FROM %s As lg \
                            WHERE lg.ath_id=%s) \
                        As f ) \
                    As fc;" % (view_name, str(ath_id))
    result = engine.execute(geojson_sql)
    for row in result:
        data = row.values()

    geojson_data = str(json.dumps(data)[1:-1])
    return geojson_data

def to_geojson_points(engine, view_name, ath_id):
    # get an output of geojson points
    geojson_sql = """
                SELECT ST_AsGeoJson(r)::json
                FROM  (
                    SELECT ST_Union(point) AS r
                    FROM %s
                    Where ath_id = %s) as foo """ % (view_name, str(ath_id))

    result = engine.execute(geojson_sql)
    for row in result:
        data = row.values()

    geojson_data = str(json.dumps(data)[1:-1])
    return geojson_data

def get_acts_centroid(engine, ath_id):
    args = """
        SELECT ST_XMax(r) AS long, ST_YMax(r) AS lat
        FROM  (
            SELECT ST_Centroid(ST_Union(Point)) AS r
            FROM "V_Stream_Activity"
            WHERE
            ath_id = %s and random() < 0.01) as foo """ % (str(ath_id))

    # Second get a list of points to draw for the heatmap as geojson
    try:
        print "getting map centroid from db..."
        ext = engine.execute(args)
        for item in ext:
            avg_long, avg_lat = item[0], item[1]
    except:
        print "error retrieving map extents!"
    return avg_long, avg_lat

def get_heatmap_points(engine, ath_id):
    args3 = """
            Select row_to_json(fc)::json
            FROM(
            SELECT array_to_json(array_agg(f))::json as points
            FROM(
                SELECT
                round(st_y(point)::numeric,3) as lt,
                round(st_x(point)::numeric,3) as lg,
                round((density)::numeric,1) as d,
                round((speed*2.23694)::numeric,1) as s,
                round((grade)::numeric,1) as g
                FROM
                "V_Point_Heatmap"
                WHERE ath_id = %s
                ) as f) as fc;""" % (str(ath_id))
    args4 = """
            Select row_to_json(fc)::json
            FROM(
            SELECT array_to_json(array_agg(f))::json as points
            FROM(
                SELECT
                round(st_y(point)::numeric,3) as lt,
                round(st_x(point)::numeric,3) as lg,
                round((density)::numeric,1) as d,
                round((speed)::numeric,1) as s,
                round((grade)::numeric,1) as g
                FROM
                "Stream_HeatPoint"
                WHERE ath_id = %s
                ) as f) as fc;""" % (str(ath_id))

    print "calculating heatmap points from db..."
    result = engine.execute(args3)
    for row in result:
        data = row.values()
    heatpoints = str(json.dumps(data)[1:-1])
    return heatpoints

def get_acts_html(engine, ath_id):
    args = """
            SELECT "act_startDate" as "Date",
                    act_type as "Type",
                    act_name as "Activity Name"
            FROM "Activity"
            WHERE ath_id = %s
            Order by act_id desc""" % (str(ath_id))
    df = pd.read_sql(args, engine)
    df.sort_values(['Date'], axis=0, ascending=False, inplace=True)
    df.index += 1
    df.index.name = 'Activity #'

    return df

def get_heatmap_lines(engine, ath_id):
    geojson_sql = """
                SELECT row_to_json(fc)
     FROM (SELECT 'FeatureCollection' As type,
                  array_to_json(array_agg(f)) As features
           FROM (SELECT 'Feature' As type,
                  st_asgeojson(lg.point, 6)::json AS geometry,
                  (
                  select row_to_json(t)
                  FROM (SELECT round((lg.density)::numeric,1) as d,
                                round((lg.speed*2.23694)::numeric,1) as s,
                                round((lg.grade)::numeric,1) as g,
                                round((lg.elev*3.28084)::numeric,0) as e,
                                round((lg.power)::numeric,0) as p,
                                round((lg.hr)::numeric,0) as h,
                                round((lg.cadence)::numeric,0) as c) as t
                                 ) as properties
                            FROM "V_Point_Heatmap" as lg WHERE ath_id = %s
                     ) as f) as fc"""  % (ath_id)

    result = engine.execute(geojson_sql)
    for row in result:
        data = row.values()

    geojson_data = str(json.dumps(data)[1:-1])
    return geojson_data<commit_msg>Iteratve over acts to remove manual and trainer before processing.<commit_after>#!/usr/bin/python
# -*- coding: utf-8 -*-
import os, json
from stravalib import client, unithelper
import pandas as pd
from datetime import datetime, timedelta
from sqlalchemy import create_engine
from unidecode import unidecode

def coerce_uni(string):
    ''' coerce ascii2 unicode for a string'''
    if string:
        string = unicode(unidecode(string))
    return string

def GetActivities(client, startDate, endDate, limit):
    # Returns a list of Strava activity objects, up to the number specified by
    # limit
    startDate = datetime.strptime(startDate, '%Y-%m-%d')
    endDate = datetime.strptime(endDate, '%Y-%m-%d')
    print "limit is: " + str(limit)
    activities = client.get_activities(before=endDate,
                                       after=startDate,
                                       limit=limit)
    return activities

def GetStreams(client, activity, types, resolution):
    # Returns a Strava 'stream', which is timeseries data from an activity
    streams = client.get_activity_streams(activity,
                                          types=types,
                                          series_type='distance',
                                          resolution=resolution)
    return streams

def DataFrame(dict, types):
    # Converts a Stream into a dataframe, and returns the dataframe
    df = pd.DataFrame()
    for item in types:
        if item in dict.keys():
            df.append(item.data)
    df.fillna('', inplace=True)
    return df

def ParseActivity(client, act, types, resolution):
    # Parses an activity from an athlete into a dataframe.  Returns the
    # dataframe
    act_id = act.id
    name = coerce_uni(act.name)
    if act.trainer==False and act.manual==False:
        streams = GetStreams(client, act_id, types, resolution)
        df = pd.DataFrame()
        # Write each row to a dataframe
        for item in types:
            if item in streams.keys():
                df[item] = pd.Series(streams[item].data, index=None)
            df['act_id'] = act.id
            df['act_startDate'] = pd.to_datetime(act.start_date_local)
    else:
       return
    return df

def get_acts_in_db(engine, table_name):
    # Return a list of already cached activities in the database
    already_dl_act_id_list = []
    try:
        args = 'SELECT Distinct act_id from "%s"' % (table_name)
        df = pd.read_sql(args, engine)
        already_dl_act_id_list = df['act_id'].tolist()
    except:
        print "no activities in database!  downloading all activities in range..."

    return already_dl_act_id_list

def check_if_new_act(act_list, already_dl_act_id_list):
    # Returns new activities not yet in the database
    for act in list(act_list):
        if act.id in already_dl_act_id_list:
            act_list.remove(act)

    return act_list

def loop_activities(client, activities, already_dl_act_id_list, types):
    # Loops through a list of activities, creating a dict of dataframes, one
    # entry for each activity
    df_lst = {}
    for act in activities:
        if act.id not in already_dl_act_id_list:
            df_lst[act.start_date] = ParseActivity(client, act, types)
    return df_lst

def calctime(time_sec, startdate):
    # calculates timestamp from the elapsed time.  Returns a datetime value.
    timestamp = startdate + timedelta(seconds=float(time_sec))

    return timestamp

def split_lat(series):
    # splits a list and retunrs the 0th item.
    lat = series[0]
    return lat

def split_long(series):
    # splits a list and retunrs the 1th item.
    long = series[1]
    return long

def create_points(lat_series, long_series):
    # Creates a string from a lat/long column to map to a Geography Point
    # datatype in PostGIS
    point_col = 'Point(' + str(long_series) + ' ' + str(lat_series) + ')'

    return point_col

def concatdf(df_lst):
    # concatenate a dictionary of dataframes into one dataframe.  Returns the
    # concat'ed dataframe
    if df_lst:
        try:
            df = pd.concat(df_lst,
                           ignore_index=False,
                           copy=False)
        except:
            print "error concatenating dataframes!"
    else:
        df = pd.DataFrame()

    return df

def cleandf(df_total):
    # Applies a number of functions to clean out a dataframe for final
    # processing
    df_total.fillna(0, inplace=True)
    df_total.reset_index(level=0, inplace=True)
    df_total['timestamp'] = map(
        calctime, df_total['time'], df_total['act_startDate'])
    df_total['timestamp'].astype('datetime64[ns]')
    df_total['lat'] = map(split_lat, (df_total['latlng']))
    df_total['long'] = map(split_long, (df_total['latlng']))
    df_total.drop(['latlng', 'act_startDate', 'index'], axis=1, inplace=True)
    df_total['point'] = map(create_points, df_total['lat'], df_total['long'])
    df_total.rename(columns={'time': 'elapsed_time',
                             'distance': 'elapsed_dist'},
                    inplace=True)

    return df_total

def process_activities(client, limit, types, engine, table_name):
    # Applies a number of functions to get and convert and clean
    activities = GetActivities(client, limit)
    #Remove trainer and manual activities
    activities = [act for act in activities if act.trainer==False and act.manual==False]
    already_dl_act_id_list = get_acts_in_db(engine, table_name)
    df_total = loop_activities(
        client, activities, already_dl_act_id_list, types)
    df_total = concatdf(df_total)

    if not df_total.empty:
        df_total = cleandf(df_total)
    else:
        print "no new data to clean"

    return df_total

def df_to_csv(df, dst_file):
    # Writes a dataframe to a csv file. Returns none
    print "writing data to csv..."
    df.to_csv(dst_file, chunksize=100)
    # except:
    #    print "error writing file to CSV!"

    return None

def zip_and_delete(src_file, dst_file):
    # Zips up and deletes a file.  Returns none.
    print "zipping and deleting file..."
    try:
        with zipfile.ZipFile(dst_file, 'w', zipfile.ZIP_DEFLATED) as myzip:
            myzip.write(src_file)
            os.remove(src_file)
    except:
        print "error zipping file!"
    return None

def fully_process_acts(client, limit, types, csv_file, zip_file, engine, table_name):
    df = process_activities(client, limit, types, engine, table_name)
    no_new_data = False
    if df.empty:
        no_new_data = True
        args = 'SELECT * from "%s"' % (table_name)
        df = pd.read_sql(args, engine)

    df_to_csv(df, csv_file)
    zip_and_delete(csv_file, zip_file)

    return df, df.ix[[0, 1, 2, 3, 4, 5]], no_new_data

def drop_table(engine, table_name):
    try:
        print "dropping table and views... " + table_name
        engine.execute('DROP TABLE "%s" CASCADE; COMMIT;' % (table_name))
        print "table dropped successfully!"
        engine.execute('DROP INDEX IF EXISTS id_geog_actid CASCADE; COMMIT;')
        print "index dropped successfully!"
    except:
        print "error dropping table! " + table_name

def df_to_postgis_db(df, engine, table_name, no_new_data):
    # Write a dataframe to a PostGIS database.  Returns none.
    if no_new_data == False:

        try:
            engine.execute("CREATE EXTENSION postgis;")
        except:
            print "postgis extension already instealled!"

        # create table for the data
        meta = MetaData(engine)
        my_table = Table(table_name, meta,
                         Column('id', Integer),
                         Column('act_startDate', DateTime),
                         Column('timestamp', DateTime),
                         Column('time', Float),
                         Column('act_id', Float),
                         Column('act_name', VARCHAR(100)),
                         Column('distance', Float),
                         Column('altitude', Float),
                         Column('velocity_smooth', Float),
                         Column('grade_smooth', Float),
                         Column('lat', Float),
                         Column('long', Float),
                         )

        Index('id_geog_actid',
              my_table.c.id,
              my_table.c.act_id)

        try:
            print "Creating table..."
            my_table.create(engine)
            print "adding point column..."
            engine.execute(
                'ALTER TABLE "%s" ADD "point" geography(POINT,4326);' % (table_name))
            print "created table " + table_name
        except:
            print "table already exists!  Appending existing table with more data..."

        print "Write data to database..."
        df.to_sql(table_name, engine,
                  if_exists='append',
                  chunksize=500,
                  index=True,
                  index_label='id')
        print "inserted data into table!"

        print "creating view..."
        my_view = 'SELECT ath_id, act_id, act_name, ST_MakeLine(st_point(long, lat) ORDER BY id) AS linestring \
                   FROM "%s" GROUP BY act_id, act_name;' % (table_name)

        try:
            engine.execute('CREATE OR REPLACE VIEW V_%s_LS AS ' %
                           (table_name) + my_view)
        except:
            print "error creating view! - HEADS UP FOR PROBLEMS!"

    return 'v_%s_ls'.lower() % (table_name)

def to_geojson_file(engine, view_name, filename):
    # check the output, this geojson isn't perfect as expected...
    geojson_sql = "SELECT row_to_json(fc) \
                   FROM (SELECT 'FeatureCollection' As type, \
                        array_to_json(array_agg(f)) As features \
                        FROM (SELECT 'Feature' As type, \
                                ST_AsGeoJSON(linestring)::json As geometry, \
                                row_to_json((SELECT l \
                                    FROM (SELECT act_id, act_name) As l)) \
                                As properties \
                            FROM %s As lg) \
                        As f ) \
                    As fc;" % (view_name)

    result = engine.execute(geojson_sql)
    for row in result:
        data = row.values()

    geojson_data = str(json.dumps(data))
    #geojson_data = [dict(row) for row in result]

    with open(filename, 'w') as f:
        try:
            f.writelines(str(geojson_data)[1:-1])
        except:
            "error exporting geojson!  Try again later :("

    return None

def to_geojson_data(engine, view_name, ath_id):
    # check the output, this geojson isn't perfect as expected...
    geojson_sql = "SELECT row_to_json(fc) \
                   FROM (SELECT 'FeatureCollection' As type, \
                        array_to_json(array_agg(f)) As features \
                        FROM (SELECT 'Feature' As type, \
                                ST_AsGeoJSON(linestring,6)::json As geometry, \
                                row_to_json((SELECT l \
                                    FROM (SELECT act_id as id, act_name as na, act_type as ty) As l)) \
                                As properties \
                            FROM %s As lg \
                            WHERE lg.ath_id=%s) \
                        As f ) \
                    As fc;" % (view_name, str(ath_id))
    result = engine.execute(geojson_sql)
    for row in result:
        data = row.values()

    geojson_data = str(json.dumps(data)[1:-1])
    return geojson_data

def to_geojson_points(engine, view_name, ath_id):
    # get an output of geojson points
    geojson_sql = """
                SELECT ST_AsGeoJson(r)::json
                FROM  (
                    SELECT ST_Union(point) AS r
                    FROM %s
                    Where ath_id = %s) as foo """ % (view_name, str(ath_id))

    result = engine.execute(geojson_sql)
    for row in result:
        data = row.values()

    geojson_data = str(json.dumps(data)[1:-1])
    return geojson_data

def get_acts_centroid(engine, ath_id):
    args = """
        SELECT ST_XMax(r) AS long, ST_YMax(r) AS lat
        FROM  (
            SELECT ST_Centroid(ST_Union(Point)) AS r
            FROM "V_Stream_Activity"
            WHERE
            ath_id = %s and random() < 0.01) as foo """ % (str(ath_id))

    # Second get a list of points to draw for the heatmap as geojson
    try:
        print "getting map centroid from db..."
        ext = engine.execute(args)
        for item in ext:
            avg_long, avg_lat = item[0], item[1]
    except:
        print "error retrieving map extents!"
    return avg_long, avg_lat

def get_heatmap_points(engine, ath_id):
    args3 = """
            Select row_to_json(fc)::json
            FROM(
            SELECT array_to_json(array_agg(f))::json as points
            FROM(
                SELECT
                round(st_y(point)::numeric,3) as lt,
                round(st_x(point)::numeric,3) as lg,
                round((density)::numeric,1) as d,
                round((speed*2.23694)::numeric,1) as s,
                round((grade)::numeric,1) as g
                FROM
                "V_Point_Heatmap"
                WHERE ath_id = %s
                ) as f) as fc;""" % (str(ath_id))
    args4 = """
            Select row_to_json(fc)::json
            FROM(
            SELECT array_to_json(array_agg(f))::json as points
            FROM(
                SELECT
                round(st_y(point)::numeric,3) as lt,
                round(st_x(point)::numeric,3) as lg,
                round((density)::numeric,1) as d,
                round((speed)::numeric,1) as s,
                round((grade)::numeric,1) as g
                FROM
                "Stream_HeatPoint"
                WHERE ath_id = %s
                ) as f) as fc;""" % (str(ath_id))

    print "calculating heatmap points from db..."
    result = engine.execute(args3)
    for row in result:
        data = row.values()
    heatpoints = str(json.dumps(data)[1:-1])
    return heatpoints

def get_acts_html(engine, ath_id):
    args = """
            SELECT "act_startDate" as "Date",
                    act_type as "Type",
                    act_name as "Activity Name"
            FROM "Activity"
            WHERE ath_id = %s
            Order by act_id desc""" % (str(ath_id))
    df = pd.read_sql(args, engine)
    df.sort_values(['Date'], axis=0, ascending=False, inplace=True)
    df.index += 1
    df.index.name = 'Activity #'

    return df

def get_heatmap_lines(engine, ath_id):
    geojson_sql = """
                SELECT row_to_json(fc)
     FROM (SELECT 'FeatureCollection' As type,
                  array_to_json(array_agg(f)) As features
           FROM (SELECT 'Feature' As type,
                  st_asgeojson(lg.point, 6)::json AS geometry,
                  (
                  select row_to_json(t)
                  FROM (SELECT round((lg.density)::numeric,1) as d,
                                round((lg.speed*2.23694)::numeric,1) as s,
                                round((lg.grade)::numeric,1) as g,
                                round((lg.elev*3.28084)::numeric,0) as e,
                                round((lg.power)::numeric,0) as p,
                                round((lg.hr)::numeric,0) as h,
                                round((lg.cadence)::numeric,0) as c) as t
                                 ) as properties
                            FROM "V_Point_Heatmap" as lg WHERE ath_id = %s
                     ) as f) as fc"""  % (ath_id)

    result = engine.execute(geojson_sql)
    for row in result:
        data = row.values()

    geojson_data = str(json.dumps(data)[1:-1])
    return geojson_data<|endoftext|>