<commit_before>#!/usr/bin/python

#######################################
## Import modules
#######################################

import sys
import subprocess
import os
import getpass
import urllib2
import time
#import math
import thread
import threading

try:
    import ephem
except:
    print "You don't have pyephem installed!"
    print "Install it like so:\n"
    print "$ sudo apt-get install python-pip"
    print "$ sudo apt-get install python-dev"
    print "$ sudo pip install pyephem"

    exit(1)

# global "constants"
REFRESH_TIME = 1 # in seconds
SAT_NAME = "ISS"
TLE_URL = "http://www.celestrak.com/NORAD/elements/stations.txt"

def getHomeDir():
   from os.path import expanduser
   return expanduser("~")

DATA_DIR  = os.path.join(getHomeDir(), ".satTracker")
TLE_FILE  = os.path.join(DATA_DIR, "tles.txt")
GRND_FILE = os.path.join(DATA_DIR, "grnd.txt")

# colors
COL_NORMAL = '\033[0m'
COL_GREY   = '\033[90m'
COL_RED    = '\033[91m'
COL_GREEN  = '\033[92m'
COL_YELLOW = '\033[93m'
COL_BLUE   = '\033[94m'
COL_PURPLE = '\033[95m'
COL_CYAN   = '\033[96m'
COL_WHITE  = '\033[97m'

# global variables
global grnd
global iss

#######################################
## Functions
#######################################

def killProgram(status):
    # This is the function that should be called to kill the program cleanly
    # if there are ever multiple threads that should be killed

    print "\nProgram is terminating."
    #thread.interrupt_main() # kills main & all daemon threads
    os._exit(status)

def setGrnd():
    # This sets the grnd global variable
    try:
        f = open(GRND_FILE, 'r')
        grndString = f.read()
        f.close()
        lines = grndString.split('\n')
        global grnd
        grnd = ephem.Observer()
        g_long = float(lines[0])
        g_lat =  float(lines[1])
        g_elev = float(lines[2])

        # Check for invalid values
        deg180 = 180 * ephem.degree
        deg90 = 90 * ephem.degree
        if g_long > deg180 or g_long < -1 * deg180:
            return 1
        if g_lat > deg90 or g_lat < -1 * deg90:
            return 1
        if g_elev < 0 or g_elev > 8848: # height of Mt. Everest
            return 1
        grnd.long = g_long
        grnd.lat =  g_lat
        grnd.elev = g_elev
        return 0
    except:
        # Error, so we need to rewrite grnd file
        return 2

def updateGrnd():

    print "Please enter information for your ground observer:"
    print "Leave a line blank to use the default value (for O.C.)."
    u_long = raw_input("Longitude (degrees): ")
    u_lat  = raw_input("Latitude (degrees): ")
    u_elev = raw_input("Elevation (m): ")

    defLong = -118.45
    defLat  = 34.0665
    defElev = 95.0

    # set values
    try:
        u_long = float(u_long)
    except:
        u_long = defLong

    try:
        u_lat = float(u_lat)
    except:
        u_lat = defLat

    try:
        u_elev = float(u_elev)
    except:
        u_elev = defElev

    # check validity of values & their types
    if u_elev < 0:
        u_elev = defElev

    # convert to radians
    u_long = u_long * ephem.degree
    u_lat = u_lat * ephem.degree

    # write values to file
    values = [ str(u_long), str(u_lat), str(u_elev), ""]
    grndString = '\n'.join(values)

    with open(GRND_FILE, 'w') as f:
        f.write(grndString)

    return

def installProgram():
    # creates the directory and asks for inputted ground observer info

    installMsg = """
    Would you like to allow issTracker to install on your computer?

    It will create on your computer:

    - a file to save TLE info for your satellites
    - a file to save longitude and latitude for your ground station
    - a directory within your home directory where these files will be saved

    """

    print installMsg
    decision = raw_input("Do you want to install issTracker? (y/n): ")
    if decision != "y":
        print "Not installing. Terminating issTracker."
        exit(1)

    print "\nInstalling issTracker\n"

    # make the directory
    dirExistsCmd = "test -d " + DATA_DIR
    if os.system(dirExistsCmd) != 0:
        if os.system("mkdir "+DATA_DIR) != 0:
            print "There was an error installing the program."
            exit(1)

    if os.system("test -f "+GRND_FILE) != 0:
        updateGrnd()

    return

def isInstalled():
    # returns True if issTracker.py appears to be installed correctly

    dirExistsCmd = "test -d " + DATA_DIR
    if os.system(dirExistsCmd) != 0:
        return False

    grndExistsCmd = "test -f " + GRND_FILE
    if os.system(grndExistsCmd) != 0:
        return False

    return True

def outputGrnd():
    # prints output for your groundstation to stdout
    g_long = grnd.long
    g_lat = grnd.lat
    g_elev = grnd.elev

    print "Printing info for your ground observer:"
    print "long:", COL_BLUE, g_long, COL_NORMAL
    print "lat: ", COL_BLUE, g_lat,  COL_NORMAL
    print "elev:", g_elev
    return

def outputSat():
    # prints output for your groundstation to stdout

    s_name = iss.name
    s_long = iss.sublong
    s_lat = iss.sublat
    s_az = iss.az
    s_alt = iss.alt
    s_elev = iss.elevation
    timeOfPass = grnd.next_pass(iss)[0]
    print s_name
    print "long:", COL_GREEN, s_long, COL_NORMAL
    print "lat: ", COL_GREEN, s_lat,  COL_NORMAL
    print "azimuth:", s_az
    print "altitude:", s_alt
    print "elevation:", s_elev
    print "next pass at" + COL_YELLOW, timeOfPass, COL_NORMAL
    return

def outputNow():
    # prints the current time according to pyephem
    print "Current time is", COL_YELLOW, ephem.now(), COL_NORMAL
    return

def updateTLE():
    # Updates the program's TLEs for satellites. It saves them on disc
    # Looks in the current directory for the TLE file

    # fetch the webpage first
    response = urllib2.urlopen(TLE_URL)
    #print "error"
    #killProgram(1)

    html = response.read() # returns a string

    # parse for the ISS (first 3 lines)
    endSub = html.find('TIANGONG')
    tleData = html[0:endSub]

    # write the data to file
    f = open(TLE_FILE, 'w')
    f.write(tleData)
    f.close()

    # reload TLEs in satellite object
    lines = tleData.split('\n')
    global iss
    iss = ephem.readtle(SAT_NAME, lines[1], lines[2])

    return

def updateVariables():
    try:
        while 1:
            now = ephem.now()
            grnd.date = now
            iss.compute(grnd) # this computes for right now relative to grnd
            #iss.compute() # this computes for right now
            time.sleep(REFRESH_TIME)
    except:
        exit(0)

def matches(a, b):
    # Takes two strings and returns True if one is a substring of the other
    # and begins at the first character of the string.

    lenA = len(a)
    lenB = len(b)

    if lenA == 0 or lenB == 0:
        # empty string should return false always
        return False

    if lenA > lenB:
        # swap them so that a is shorter
        tmp = b
        b = a
        a = tmp

    # assume that lenA <= lenB
    b = b[0:lenA]

    # if a & b are a match, then return true
    return a == b

def usage():
    HELP_MSG="""
To enter a command to issTracker, enter one or more characters at the start
of the desired command option.

issTracker command prompt options:

quit                             This quits the application cleanly
help                             This displays this help message
clear                            Clear the screen
update                           Update the ISS TLE automatically
grnd, ground                     Display ground station information
now                              Display the current time in UTC
change                           Enter new ground station information
print (or simply hitting enter)  Display ISS location and time of next pass
"""
    print HELP_MSG
    return

def prompt():
    try:
        while 1:
            key = raw_input("\nPress enter to see values, q to quit: ")
            if matches(key,"quit") or key == "Q" or key == ";q" or key == "exit":
                killProgram(0)
                sys.exit(0) # redundant, but safe
            elif matches(key,"help") or key == "--help":
                usage()
            elif matches(key,"clear") or key == "cls":
                os.system('clear');
            elif matches(key,"update"):
                updateTLE()
                print "Update is complete"
            elif matches(key,"grnd") or key == "ground":
                outputGrnd()
            elif matches(key,"now"):
                outputNow()
            elif matches(key,"change"):
                updateGrnd()
                ret = setGrnd()
                if ret == 0:
                    print "Your update of ground station info is complete."
                    outputGrnd()
                else:
                    print "It appears there was an error with your grnd values."
                    killProgram(1)
            else:
                outputSat()
    except:
        exit(0)

def main():
    ## Check if installed
    if not isInstalled():
        installProgram()

    ## initialize satellite info
    # pull the TLE from disc
    try:
        f = open(TLE_FILE, 'r')
        tleString = f.read()
        f.close()
        lines = tleString.split('\n')
        global iss
        iss = ephem.readtle(SAT_NAME, lines[1], lines[2])
    except:
        # there was an error with the file format or the file did not exist
        updateTLE()

    ## set up ground station information
    while True:
        ret = setGrnd()
        if ret == 0:
            break
        else:
            # there was an error with the file format or the file did not exist
            print COL_RED, "Error with grnd file.", COL_NORMAL, "Please update it with valid information."
            updateGrnd()

    #outputGrnd()

    # use threading module to spawn new threads
    my_threads = list()
    my_threads.append(threading.Thread(target=updateVariables) )
    my_threads.append(threading.Thread(target=prompt) )
    my_threads[0].daemon = True # run this thread in the background
    my_threads[1].daemon = False
    my_threads[0].start()
    my_threads[1].start()

    for t in my_threads:
        while t.isAlive():
            t.join(2)

    try:
        while 1:
            pass
        killProgram(0)
    except:
        killProgram(0)

if __name__ == "__main__":
    # execute the main function now
    main()

exit(0)
<commit_msg>Removed one line to tighten it up a bit.<commit_after>#!/usr/bin/python

#######################################
## Import modules
#######################################

import sys
import subprocess
import os
import getpass
import urllib2
import time
#import math
import thread
import threading

try:
    import ephem
except:
    print "You don't have pyephem installed!"
    print "Install it like so:\n"
    print "$ sudo apt-get install python-pip"
    print "$ sudo apt-get install python-dev"
    print "$ sudo pip install pyephem"

    exit(1)

# global "constants"
REFRESH_TIME = 1 # in seconds
SAT_NAME = "ISS"
TLE_URL = "http://www.celestrak.com/NORAD/elements/stations.txt"

def getHomeDir():
   return os.path.expanduser("~")

DATA_DIR  = os.path.join(getHomeDir(), ".satTracker")
TLE_FILE  = os.path.join(DATA_DIR, "tles.txt")
GRND_FILE = os.path.join(DATA_DIR, "grnd.txt")

# colors
COL_NORMAL = '\033[0m'
COL_GREY   = '\033[90m'
COL_RED    = '\033[91m'
COL_GREEN  = '\033[92m'
COL_YELLOW = '\033[93m'
COL_BLUE   = '\033[94m'
COL_PURPLE = '\033[95m'
COL_CYAN   = '\033[96m'
COL_WHITE  = '\033[97m'

# global variables
global grnd
global iss

#######################################
## Functions
#######################################

def killProgram(status):
    # This is the function that should be called to kill the program cleanly
    # if there are ever multiple threads that should be killed

    print "\nProgram is terminating."
    #thread.interrupt_main() # kills main & all daemon threads
    os._exit(status)

def setGrnd():
    # This sets the grnd global variable
    try:
        f = open(GRND_FILE, 'r')
        grndString = f.read()
        f.close()
        lines = grndString.split('\n')
        global grnd
        grnd = ephem.Observer()
        g_long = float(lines[0])
        g_lat =  float(lines[1])
        g_elev = float(lines[2])

        # Check for invalid values
        deg180 = 180 * ephem.degree
        deg90 = 90 * ephem.degree
        if g_long > deg180 or g_long < -1 * deg180:
            return 1
        if g_lat > deg90 or g_lat < -1 * deg90:
            return 1
        if g_elev < 0 or g_elev > 8848: # height of Mt. Everest
            return 1
        grnd.long = g_long
        grnd.lat =  g_lat
        grnd.elev = g_elev
        return 0
    except:
        # Error, so we need to rewrite grnd file
        return 2

def updateGrnd():

    print "Please enter information for your ground observer:"
    print "Leave a line blank to use the default value (for O.C.)."
    u_long = raw_input("Longitude (degrees): ")
    u_lat  = raw_input("Latitude (degrees): ")
    u_elev = raw_input("Elevation (m): ")

    defLong = -118.45
    defLat  = 34.0665
    defElev = 95.0

    # set values
    try:
        u_long = float(u_long)
    except:
        u_long = defLong

    try:
        u_lat = float(u_lat)
    except:
        u_lat = defLat

    try:
        u_elev = float(u_elev)
    except:
        u_elev = defElev

    # check validity of values & their types
    if u_elev < 0:
        u_elev = defElev

    # convert to radians
    u_long = u_long * ephem.degree
    u_lat = u_lat * ephem.degree

    # write values to file
    values = [ str(u_long), str(u_lat), str(u_elev), ""]
    grndString = '\n'.join(values)

    with open(GRND_FILE, 'w') as f:
        f.write(grndString)

    return

def installProgram():
    # creates the directory and asks for inputted ground observer info

    installMsg = """
    Would you like to allow issTracker to install on your computer?

    It will create on your computer:

    - a file to save TLE info for your satellites
    - a file to save longitude and latitude for your ground station
    - a directory within your home directory where these files will be saved

    """

    print installMsg
    decision = raw_input("Do you want to install issTracker? (y/n): ")
    if decision != "y":
        print "Not installing. Terminating issTracker."
        exit(1)

    print "\nInstalling issTracker\n"

    # make the directory
    dirExistsCmd = "test -d " + DATA_DIR
    if os.system(dirExistsCmd) != 0:
        if os.system("mkdir "+DATA_DIR) != 0:
            print "There was an error installing the program."
            exit(1)

    if os.system("test -f "+GRND_FILE) != 0:
        updateGrnd()

    return

def isInstalled():
    # returns True if issTracker.py appears to be installed correctly

    dirExistsCmd = "test -d " + DATA_DIR
    if os.system(dirExistsCmd) != 0:
        return False

    grndExistsCmd = "test -f " + GRND_FILE
    if os.system(grndExistsCmd) != 0:
        return False

    return True

def outputGrnd():
    # prints output for your groundstation to stdout
    g_long = grnd.long
    g_lat = grnd.lat
    g_elev = grnd.elev

    print "Printing info for your ground observer:"
    print "long:", COL_BLUE, g_long, COL_NORMAL
    print "lat: ", COL_BLUE, g_lat,  COL_NORMAL
    print "elev:", g_elev
    return

def outputSat():
    # prints output for your groundstation to stdout

    s_name = iss.name
    s_long = iss.sublong
    s_lat = iss.sublat
    s_az = iss.az
    s_alt = iss.alt
    s_elev = iss.elevation
    timeOfPass = grnd.next_pass(iss)[0]
    print s_name
    print "long:", COL_GREEN, s_long, COL_NORMAL
    print "lat: ", COL_GREEN, s_lat,  COL_NORMAL
    print "azimuth:", s_az
    print "altitude:", s_alt
    print "elevation:", s_elev
    print "next pass at" + COL_YELLOW, timeOfPass, COL_NORMAL
    return

def outputNow():
    # prints the current time according to pyephem
    print "Current time is", COL_YELLOW, ephem.now(), COL_NORMAL
    return

def updateTLE():
    # Updates the program's TLEs for satellites. It saves them on disc
    # Looks in the current directory for the TLE file

    # fetch the webpage first
    response = urllib2.urlopen(TLE_URL)
    #print "error"
    #killProgram(1)

    html = response.read() # returns a string

    # parse for the ISS (first 3 lines)
    endSub = html.find('TIANGONG')
    tleData = html[0:endSub]

    # write the data to file
    f = open(TLE_FILE, 'w')
    f.write(tleData)
    f.close()

    # reload TLEs in satellite object
    lines = tleData.split('\n')
    global iss
    iss = ephem.readtle(SAT_NAME, lines[1], lines[2])

    return

def updateVariables():
    try:
        while 1:
            now = ephem.now()
            grnd.date = now
            iss.compute(grnd) # this computes for right now relative to grnd
            #iss.compute() # this computes for right now
            time.sleep(REFRESH_TIME)
    except:
        exit(0)

def matches(a, b):
    # Takes two strings and returns True if one is a substring of the other
    # and begins at the first character of the string.

    lenA = len(a)
    lenB = len(b)

    if lenA == 0 or lenB == 0:
        # empty string should return false always
        return False

    if lenA > lenB:
        # swap them so that a is shorter
        tmp = b
        b = a
        a = tmp

    # assume that lenA <= lenB
    b = b[0:lenA]

    # if a & b are a match, then return true
    return a == b

def usage():
    HELP_MSG="""
To enter a command to issTracker, enter one or more characters at the start
of the desired command option.

issTracker command prompt options:

quit                             This quits the application cleanly
help                             This displays this help message
clear                            Clear the screen
update                           Update the ISS TLE automatically
grnd, ground                     Display ground station information
now                              Display the current time in UTC
change                           Enter new ground station information
print (or simply hitting enter)  Display ISS location and time of next pass
"""
    print HELP_MSG
    return

def prompt():
    try:
        while 1:
            key = raw_input("\nPress enter to see values, q to quit: ")
            if matches(key,"quit") or key == "Q" or key == ";q" or key == "exit":
                killProgram(0)
                sys.exit(0) # redundant, but safe
            elif matches(key,"help") or key == "--help":
                usage()
            elif matches(key,"clear") or key == "cls":
                os.system('clear');
            elif matches(key,"update"):
                updateTLE()
                print "Update is complete"
            elif matches(key,"grnd") or key == "ground":
                outputGrnd()
            elif matches(key,"now"):
                outputNow()
            elif matches(key,"change"):
                updateGrnd()
                ret = setGrnd()
                if ret == 0:
                    print "Your update of ground station info is complete."
                    outputGrnd()
                else:
                    print "It appears there was an error with your grnd values."
                    killProgram(1)
            else:
                outputSat()
    except:
        exit(0)

def main():
    ## Check if installed
    if not isInstalled():
        installProgram()

    ## initialize satellite info
    # pull the TLE from disc
    try:
        f = open(TLE_FILE, 'r')
        tleString = f.read()
        f.close()
        lines = tleString.split('\n')
        global iss
        iss = ephem.readtle(SAT_NAME, lines[1], lines[2])
    except:
        # there was an error with the file format or the file did not exist
        updateTLE()

    ## set up ground station information
    while True:
        ret = setGrnd()
        if ret == 0:
            break
        else:
            # there was an error with the file format or the file did not exist
            print COL_RED, "Error with grnd file.", COL_NORMAL, "Please update it with valid information."
            updateGrnd()

    #outputGrnd()

    # use threading module to spawn new threads
    my_threads = list()
    my_threads.append(threading.Thread(target=updateVariables) )
    my_threads.append(threading.Thread(target=prompt) )
    my_threads[0].daemon = True # run this thread in the background
    my_threads[1].daemon = False
    my_threads[0].start()
    my_threads[1].start()

    for t in my_threads:
        while t.isAlive():
            t.join(2)

    try:
        while 1:
            pass
        killProgram(0)
    except:
        killProgram(0)

if __name__ == "__main__":
    # execute the main function now
    main()

exit(0)
<|endoftext|>


<commit_before>import tkinter as tk
import pygubu
import numpy as np
import matwidget
import urllib.request as re
import csv
import requests

class googlefinance:

	def __init__(self, master):
		# Root definition
		self.master = master # De facto self.root
		self.builder = builder = pygubu.Builder() # This is required to pull the different objects
		builder.add_from_file("gui.ui") # Adds the gui.ui file, such that we may use it.
		self.mainwindow = builder.get_object('root', master) # This is the master frame in the Tk.

		# The get_object("name") allows to have a variable x point to the object (e.g. frame),
		# so as to allow further manipulation

		self.path  = builder.get_object("pathchooserinput1")

		# We name the graph frame self.graph, such that we can parse it into the matwidget function
		# which in turn paints the matplotlib canvas in the given frame.
		self.graph = builder.get_object("graph")
		self.reqdata()
		matwidget.graph(self.graph, self.data)

	def urls(self):
		# TODO
		pass

	def reqdata(self):
		self.yahoo = 'http://ichart.finance.yahoo.com/table.csv?s=AAPL&c=1962'
		with requests.Session() as s:
			download = s.get(self.yahoo)
			decoded_content = download.content.decode('utf-8')
			cr = csv.reader(decoded_content.splitlines(), delimiter=',')
			self.data = list(cr)

if __name__ == '__main__':
	root = tk.Tk()
	app = googlefinance(root)
	root.mainloop()
<commit_msg>Delete main.py<commit_after><|endoftext|>


<commit_before>#!/usr/bin/env python

import requests

class jamendo(object):

    def __init__(self, client_id, api_defaults={}, file_defaults={}):

        self.base_url = "http://api.jamendo.com"
        self.api_version = "v3.0"
        self.api_url = "%s/%s" % (self.base_url, self.api_version)
        self.file_endpoint = "tracks/file"

        self.default_api_payload = dict(
            [("client_id", client_id), ("format", "json")] +
            api_defaults.items()
        )

        self.default_file_payload = dict(
            [("client_id", client_id), ("audioformat", "ogg")] +
            file_defaults.items()
        )

    def get(self, data_type, args={}):
        url = "%s/%s" % (self.api_url, data_type)
        payload = dict(
            self.default_api_payload.items() +
            args.items()
        )
        r = requests.get(url, params=payload)
        return r.json()["results"]

    def download_track(self, track_id, args={}):
        url = "%s/%s" % (self.url, self.file_endpoint)
        payload = dict(
            self.default_file_payload.items() +
            args.items() +
            [("id", track_id)]
        )
        r = requests.get(url, params=payload)
        return r

if __name__ == '__main__':
    import unittest
    from tests.testjamendoapi import Testjamendo
    Testjamendo.header()
    unittest.main()
<commit_msg>download_track method just returns binary data<commit_after>#!/usr/bin/env python

import requests

class jamendo(object):

    def __init__(self, client_id, api_defaults={}, file_defaults={}):

        self.base_url = "http://api.jamendo.com"
        self.api_version = "v3.0"
        self.api_url = "%s/%s" % (self.base_url, self.api_version)
        self.file_endpoint = "tracks/file"

        self.default_api_payload = dict(
            [("client_id", client_id), ("format", "json")] +
            api_defaults.items()
        )

        self.default_file_payload = dict(
            [("client_id", client_id), ("audioformat", "ogg")] +
            file_defaults.items()
        )

    def get(self, data_type, args={}):
        url = "%s/%s" % (self.api_url, data_type)
        payload = dict(
            self.default_api_payload.items() +
            args.items()
        )
        r = requests.get(url, params=payload)
        return r.json()["results"]

    def download_track(self, track_id, args={}):
        url = "%s/%s" % (self.url, self.file_endpoint)
        payload = dict(
            self.default_file_payload.items() +
            args.items() +
            [("id", track_id)]
        )
        r = requests.get(url, params=payload)
        return r.content()

if __name__ == '__main__':
    import unittest
    from tests.testjamendoapi import Testjamendo
    Testjamendo.header()
    unittest.main()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

import json
import logging
import os

try:
    import queue
except ImportError:
    import Queue as queue
try:
    import socketserver
except ImportError:
    import SocketServer as socketserver

from apscheduler.schedulers.background import BackgroundScheduler
from datetime import datetime, timedelta
from flask import Flask, render_template, Response, request, abort
from flask_assets import Environment, Bundle
from flask.templating import TemplateNotFound
from jobs import load_jobs
from random import randint

app = Flask(__name__)
app.logger.setLevel(logging.INFO)
app.jinja_env.trim_blocks = True
app.jinja_env.lstrip_blocks = True
sched = BackgroundScheduler(logger=app.logger)
queues = {}
last_events = {}

@app.before_first_request
def _configure_bundles():
    js = ['main.js']
    css = ['main.css']
    widgets_path = os.path.abspath(os.path.join(os.path.dirname(__file__),
                                                'static', 'widgets'))

    for widget in os.listdir(widgets_path):
        widget_path = os.path.join('widgets', widget)
        for asset_file in os.listdir(os.path.join(widgets_path, widget)):
            asset_path = os.path.join(widget_path, asset_file)
            if asset_file.endswith('.js'):
                js.append(asset_path)
            elif asset_file.endswith('.css'):
                css.append(asset_path)

    assets = Environment(app)
    if app.debug:
        assets.register('js_all', Bundle(*js, output='gen/app.js'))
        assets.register('css_all', Bundle(*css, output='gen/styles.css'))
    else:
        assets.register('js_min_all', Bundle(*js, filters='rjsmin',
                                             output='gen/app.min.js'))
        assets.register('css_min_all', Bundle(*css, filters='cssmin',
                                              output='gen/styles.min.css'))

@app.route('/w/<job_id>')
@app.route('/widget/<job_id>')
def widget(job_id):
    if not _is_enabled(job_id):
        abort(404)
    x = request.args.get('x', 3)
    widgets = _enabled_jobs()
    # Widget name shares the name of the job implementation
    widget = _config()['JOBS'][job_id].get('job_impl', job_id)
    return render_template('index.html', layout='layout_single.html',
                           widget=widget, job=job_id, x=x, widgets=widgets)

@app.route('/')
@app.route('/d/<layout>')
@app.route('/dashboard/<layout>')
def dashboard(layout=None):
    locale = request.args.get('locale')
    widgets = _enabled_jobs()
    layout = layout or _config().get('DEFAULT_LAYOUT')
    if layout is None:
        return render_template('index.html', locale=locale, widgets=widgets)
    try:
        return render_template('index.html',
                               layout='layouts/{0}.html'.format(layout),
                               locale=locale, widgets=widgets)
    except TemplateNotFound:
        abort(404)

@app.route('/events')
def events():
    remote_port = request.environ['REMOTE_PORT']
    current_queue = queue.Queue()
    queues[remote_port] = current_queue

    for event in last_events.values():
        current_queue.put(event)

    def consume():
        while True:
            data = current_queue.get()
            if data is None:
                break
            yield 'data: %s\n\n' % (data,)

    response = Response(consume(), mimetype='text/event-stream')
    response.headers['X-Accel-Buffering'] = 'no'
    return response

@app.route('/events/<job_id>', methods=['POST'])
def create_event(job_id):
    if not _is_enabled(job_id):
        abort(404)
    body = request.get_json()
    if not body:
        abort(400)
    _add_event(job_id, body)
    return '', 201

def _config():
    if set(['DEFAULT_LAYOUT', 'JOBS']).issubset(app.config.keys()):
        return app.config
    app.config.from_envvar('JARVIS_SETTINGS')
    return app.config

def _enabled_jobs():
    config = _config()['JOBS']
    return [job_id for job_id in config.keys()
            if config[job_id].get('enabled')]

def _is_enabled(job_id):
    return job_id in _enabled_jobs()

@app.context_processor
def _inject_template_methods():
    return dict(is_job_enabled=_is_enabled)

@app.before_first_request
def _schedule_jobs():
    offset = 0
    jobs = load_jobs()

    for job_id, job_config in _config()['JOBS'].items():
        job_impl = job_config.get('job_impl', job_id)
        if not job_config.get('enabled'):
            app.logger.info('Skipping disabled job: %s', job_id)
            continue
        if job_impl not in jobs:
            app.logger.info(('Skipping job with ID %s (no such '
                             'implementation: %s)'), job_id, job_impl)
            continue
        job = jobs[job_impl](job_config)
        if app.debug:
            start_date = datetime.now() + timedelta(seconds=1)
        else:
            offset += randint(4, 10)
            start_date = datetime.now() + timedelta(seconds=offset)

        job.start_date = start_date
        app.logger.info('Scheduling job with ID %s (implementation: %s): %s',
                        job_id, job_impl, job)
        sched.add_job(_run_job,
                      'interval',
                      name=job_id,
                      next_run_time=job.start_date,
                      coalesce=True,
                      seconds=job.interval,
                      kwargs={'job_id': job_id, 'job': job})
    if not sched.running:
        sched.start()

def _add_event(job_id, data):
    json_data = json.dumps({
        'body': data,
        'job': job_id
    }, separators=(',', ':'), sort_keys=True)
    last_events[job_id] = json_data
    for q in queues.values():
        q.put(json_data)

def _run_job(job_id, job):
    try:
        data = job.get()
        _add_event(job_id, data)
    except Exception:
        app.logger.exception('Failed to execute job with ID: ' + job_id)

def _close_stream(*args, **kwargs):
    remote_port = args[2][1]
    if remote_port in queues:
        del queues[remote_port]

socketserver.BaseServer.handle_error = _close_stream
<commit_msg>Always load config from environment when not testing<commit_after># -*- coding: utf-8 -*-

import json
import logging
import os

try:
    import queue
except ImportError:
    import Queue as queue
try:
    import socketserver
except ImportError:
    import SocketServer as socketserver

from apscheduler.schedulers.background import BackgroundScheduler
from datetime import datetime, timedelta
from flask import Flask, render_template, Response, request, abort
from flask_assets import Environment, Bundle
from flask.templating import TemplateNotFound
from jobs import load_jobs
from random import randint

app = Flask(__name__)
app.logger.setLevel(logging.INFO)
app.jinja_env.trim_blocks = True
app.jinja_env.lstrip_blocks = True
sched = BackgroundScheduler(logger=app.logger)
queues = {}
last_events = {}

@app.before_first_request
def _configure_bundles():
    js = ['main.js']
    css = ['main.css']
    widgets_path = os.path.abspath(os.path.join(os.path.dirname(__file__),
                                                'static', 'widgets'))

    for widget in os.listdir(widgets_path):
        widget_path = os.path.join('widgets', widget)
        for asset_file in os.listdir(os.path.join(widgets_path, widget)):
            asset_path = os.path.join(widget_path, asset_file)
            if asset_file.endswith('.js'):
                js.append(asset_path)
            elif asset_file.endswith('.css'):
                css.append(asset_path)

    assets = Environment(app)
    if app.debug:
        assets.register('js_all', Bundle(*js, output='gen/app.js'))
        assets.register('css_all', Bundle(*css, output='gen/styles.css'))
    else:
        assets.register('js_min_all', Bundle(*js, filters='rjsmin',
                                             output='gen/app.min.js'))
        assets.register('css_min_all', Bundle(*css, filters='cssmin',
                                              output='gen/styles.min.css'))

@app.route('/w/<job_id>')
@app.route('/widget/<job_id>')
def widget(job_id):
    if not _is_enabled(job_id):
        abort(404)
    x = request.args.get('x', 3)
    widgets = _enabled_jobs()
    # Widget name shares the name of the job implementation
    widget = _config()['JOBS'][job_id].get('job_impl', job_id)
    return render_template('index.html', layout='layout_single.html',
                           widget=widget, job=job_id, x=x, widgets=widgets)

@app.route('/')
@app.route('/d/<layout>')
@app.route('/dashboard/<layout>')
def dashboard(layout=None):
    locale = request.args.get('locale')
    widgets = _enabled_jobs()
    layout = layout or _config().get('DEFAULT_LAYOUT')
    if layout is None:
        return render_template('index.html', locale=locale, widgets=widgets)
    try:
        return render_template('index.html',
                               layout='layouts/{0}.html'.format(layout),
                               locale=locale, widgets=widgets)
    except TemplateNotFound:
        abort(404)

@app.route('/events')
def events():
    remote_port = request.environ['REMOTE_PORT']
    current_queue = queue.Queue()
    queues[remote_port] = current_queue

    for event in last_events.values():
        current_queue.put(event)

    def consume():
        while True:
            data = current_queue.get()
            if data is None:
                break
            yield 'data: %s\n\n' % (data,)

    response = Response(consume(), mimetype='text/event-stream')
    response.headers['X-Accel-Buffering'] = 'no'
    return response

@app.route('/events/<job_id>', methods=['POST'])
def create_event(job_id):
    if not _is_enabled(job_id):
        abort(404)
    body = request.get_json()
    if not body:
        abort(400)
    _add_event(job_id, body)
    return '', 201

def _config():
    if app.testing:  # tests set their own config
        return app.config
    app.config.from_envvar('JARVIS_SETTINGS')
    return app.config

def _enabled_jobs():
    config = _config()['JOBS']
    return [job_id for job_id in config.keys()
            if config[job_id].get('enabled')]

def _is_enabled(job_id):
    return job_id in _enabled_jobs()

@app.context_processor
def _inject_template_methods():
    return dict(is_job_enabled=_is_enabled)

@app.before_first_request
def _schedule_jobs():
    offset = 0
    jobs = load_jobs()

    for job_id, job_config in _config()['JOBS'].items():
        job_impl = job_config.get('job_impl', job_id)
        if not job_config.get('enabled'):
            app.logger.info('Skipping disabled job: %s', job_id)
            continue
        if job_impl not in jobs:
            app.logger.info(('Skipping job with ID %s (no such '
                             'implementation: %s)'), job_id, job_impl)
            continue
        job = jobs[job_impl](job_config)
        if app.debug:
            start_date = datetime.now() + timedelta(seconds=1)
        else:
            offset += randint(4, 10)
            start_date = datetime.now() + timedelta(seconds=offset)

        job.start_date = start_date
        app.logger.info('Scheduling job with ID %s (implementation: %s): %s',
                        job_id, job_impl, job)
        sched.add_job(_run_job,
                      'interval',
                      name=job_id,
                      next_run_time=job.start_date,
                      coalesce=True,
                      seconds=job.interval,
                      kwargs={'job_id': job_id, 'job': job})
    if not sched.running:
        sched.start()

def _add_event(job_id, data):
    json_data = json.dumps({
        'body': data,
        'job': job_id
    }, separators=(',', ':'), sort_keys=True)
    last_events[job_id] = json_data
    for q in queues.values():
        q.put(json_data)

def _run_job(job_id, job):
    try:
        data = job.get()
        _add_event(job_id, data)
    except Exception:
        app.logger.exception('Failed to execute job with ID: ' + job_id)

def _close_stream(*args, **kwargs):
    remote_port = args[2][1]
    if remote_port in queues:
        del queues[remote_port]

socketserver.BaseServer.handle_error = _close_stream
<|endoftext|>


<commit_before>from asyncio import gather, get_event_loop, sleep
from io import BytesIO

from jd4.api import VJ4Session
from jd4.case import read_legacy_cases
from jd4.cache import cache_open, cache_invalidate
from jd4.cgroup import try_init_cgroup
from jd4.config import config
from jd4.langs import langs
from jd4.log import logger
from jd4.sandbox import create_sandbox
from jd4.status import STATUS_COMPILE_ERROR, STATUS_SYSTEM_ERROR

RETRY_DELAY_SEC = 30

try_init_cgroup()
loop = get_event_loop()
sandbox = loop.run_until_complete(create_sandbox())

class CompileError(Exception):
    pass

class SystemError(Exception):
    pass

class JudgeHandler:
    def __init__(self, session, request, ws):
        self.session = session
        self.request = request
        self.ws = ws
        self.tag = self.request.pop('tag')

    async def handle(self):
        try:
            type = self.request.pop('type')
            if type == 0:
                await self.submission()
            elif type == 1:
                await self.pretest()
            else:
                raise SystemError('Unsupported type: {}'.format(type))
            for key in self.request:
                logger.warning('Unused key in judge request: %s', key)
        except CompileError as e:
            self.next(compiler_text=str(e))
            self.end(status=STATUS_COMPILE_ERROR, score=0, time_ms=0, memory_kb=0)
        except Exception as e:
            logger.exception(e)
            self.next(compiler_text=repr(e))
            self.end(status=STATUS_SYSTEM_ERROR, score=0, time_ms=0, memory_kb=0)

    async def submission(self):
        domain_id = self.request.pop('domain_id')
        pid = self.request.pop('pid')
        rid = self.request.pop('rid')
        logger.info('Submission: %s, %s, %s', domain_id, pid, rid)
        cases_file, package = await gather(cache_open(self.session, domain_id, pid), self.build())
        try:
            await self.judge(cases_file, package)
        finally:
            cases_file.close()

    async def pretest(self):
        domain_id = self.request.pop('domain_id')
        pid = self.request.pop('pid')
        rid = self.request.pop('rid')
        logger.info('Pretest: %s, %s, %s', domain_id, pid, rid)
        cases_data, package = await gather(self.session.record_pretest_data(rid), self.build())
        await self.judge(BytesIO(cases_data), package)

    async def build(self):
        lang = self.request.pop('lang')
        build_fn = langs.get(lang)
        if not build_fn:
            raise SystemError('Unsupported language: {}'.format(lang))
        package, message = await build_fn(sandbox, self.request.pop('code').encode())
        if not package:
            logger.info('Compile error: %s', message)
            raise CompileError(message)
        return package

    async def judge(self, cases_file, package):
        cases = list(read_legacy_cases(cases_file))
        total_status = 0
        total_score = 0
        total_time_usage_ns = 0
        total_memory_usage_bytes = 0
        for index, case in enumerate(cases):
            status, score, time_usage_ns, memory_usage_bytes, stderr = await case.judge(sandbox, package)
            self.next(case={'status': status,
                            'score': score,
                            'time_ms': time_usage_ns // 1000000,
                            'memory_kb': memory_usage_bytes // 1024,
                            'judge_text': stderr.decode(encoding='utf-8', errors='replace')},
                      progress=(index + 1) / len(cases))
            logger.debug('Case %d: %d, %g, %g, %g, %s',
                         index, status, score, time_usage_ns / 1000000, memory_usage_bytes / 1024, stderr)
            total_status = max(total_status, status)
            total_score += score
            total_time_usage_ns += time_usage_ns
            total_memory_usage_bytes = max(total_memory_usage_bytes, memory_usage_bytes)
        self.end(status=total_status,
                 score=total_score,
                 time_ms=total_time_usage_ns // 1000000,
                 memory_kb=total_memory_usage_bytes // 1024)
        logger.info('Total: %d, %g, %g, %g',
                     total_status, total_score, total_time_usage_ns / 1000000, total_memory_usage_bytes / 1024)

    def next(self, **kwargs):
        self.ws.send_json({'key': 'next', 'tag': self.tag, **kwargs})

    def end(self, **kwargs):
        self.ws.send_json({'key': 'end', 'tag': self.tag, **kwargs})

async def daemon():
    async with VJ4Session() as session:
        while True:
            try:
                await session.login_if_needed(config['uname'], config['password'])
                await session.judge_consume(JudgeHandler)
            except Exception as e:
                logger.exception(e)
            logger.info('Retrying after %d seconds', RETRY_DELAY_SEC)
            await sleep(RETRY_DELAY_SEC)

# TODO(iceboy): maintain cache (invalidate, background download).
loop.run_until_complete(daemon())
<commit_msg>fix progress<commit_after>from asyncio import gather, get_event_loop, sleep
from io import BytesIO

from jd4.api import VJ4Session
from jd4.case import read_legacy_cases
from jd4.cache import cache_open, cache_invalidate
from jd4.cgroup import try_init_cgroup
from jd4.config import config
from jd4.langs import langs
from jd4.log import logger
from jd4.sandbox import create_sandbox
from jd4.status import STATUS_COMPILE_ERROR, STATUS_SYSTEM_ERROR

RETRY_DELAY_SEC = 30

try_init_cgroup()
loop = get_event_loop()
sandbox = loop.run_until_complete(create_sandbox())

class CompileError(Exception):
    pass

class SystemError(Exception):
    pass

class JudgeHandler:
    def __init__(self, session, request, ws):
        self.session = session
        self.request = request
        self.ws = ws
        self.tag = self.request.pop('tag')

    async def handle(self):
        try:
            type = self.request.pop('type')
            if type == 0:
                await self.submission()
            elif type == 1:
                await self.pretest()
            else:
                raise SystemError('Unsupported type: {}'.format(type))
            for key in self.request:
                logger.warning('Unused key in judge request: %s', key)
        except CompileError as e:
            self.next(compiler_text=str(e))
            self.end(status=STATUS_COMPILE_ERROR, score=0, time_ms=0, memory_kb=0)
        except Exception as e:
            logger.exception(e)
            self.next(compiler_text=repr(e))
            self.end(status=STATUS_SYSTEM_ERROR, score=0, time_ms=0, memory_kb=0)

    async def submission(self):
        domain_id = self.request.pop('domain_id')
        pid = self.request.pop('pid')
        rid = self.request.pop('rid')
        logger.info('Submission: %s, %s, %s', domain_id, pid, rid)
        cases_file, package = await gather(cache_open(self.session, domain_id, pid), self.build())
        try:
            await self.judge(cases_file, package)
        finally:
            cases_file.close()

    async def pretest(self):
        domain_id = self.request.pop('domain_id')
        pid = self.request.pop('pid')
        rid = self.request.pop('rid')
        logger.info('Pretest: %s, %s, %s', domain_id, pid, rid)
        cases_data, package = await gather(self.session.record_pretest_data(rid), self.build())
        await self.judge(BytesIO(cases_data), package)

    async def build(self):
        lang = self.request.pop('lang')
        build_fn = langs.get(lang)
        if not build_fn:
            raise SystemError('Unsupported language: {}'.format(lang))
        package, message = await build_fn(sandbox, self.request.pop('code').encode())
        if not package:
            logger.info('Compile error: %s', message)
            raise CompileError(message)
        return package

    async def judge(self, cases_file, package):
        cases = list(read_legacy_cases(cases_file))
        total_status = 0
        total_score = 0
        total_time_usage_ns = 0
        total_memory_usage_bytes = 0
        for index, case in enumerate(cases):
            status, score, time_usage_ns, memory_usage_bytes, stderr = await case.judge(sandbox, package)
            self.next(case={'status': status,
                            'score': score,
                            'time_ms': time_usage_ns // 1000000,
                            'memory_kb': memory_usage_bytes // 1024,
                            'judge_text': stderr.decode(encoding='utf-8', errors='replace')},
                      progress=(index + 1) * 100 // len(cases))
            logger.debug('Case %d: %d, %g, %g, %g, %s',
                         index, status, score, time_usage_ns / 1000000, memory_usage_bytes / 1024, stderr)
            total_status = max(total_status, status)
            total_score += score
            total_time_usage_ns += time_usage_ns
            total_memory_usage_bytes = max(total_memory_usage_bytes, memory_usage_bytes)
        self.end(status=total_status,
                 score=total_score,
                 time_ms=total_time_usage_ns // 1000000,
                 memory_kb=total_memory_usage_bytes // 1024)
        logger.info('Total: %d, %g, %g, %g',
                     total_status, total_score, total_time_usage_ns / 1000000, total_memory_usage_bytes / 1024)

    def next(self, **kwargs):
        self.ws.send_json({'key': 'next', 'tag': self.tag, **kwargs})

    def end(self, **kwargs):
        self.ws.send_json({'key': 'end', 'tag': self.tag, **kwargs})

async def daemon():
    async with VJ4Session() as session:
        while True:
            try:
                await session.login_if_needed(config['uname'], config['password'])
                await session.judge_consume(JudgeHandler)
            except Exception as e:
                logger.exception(e)
            logger.info('Retrying after %d seconds', RETRY_DELAY_SEC)
            await sleep(RETRY_DELAY_SEC)

# TODO(iceboy): maintain cache (invalidate, background download).
loop.run_until_complete(daemon())
<|endoftext|>


<commit_before>from jedi._compatibility import encoding, is_py3
import inspect
import os
import time

try:
    if os.name == 'nt':
        # does not work on Windows, as pyreadline and colorama interfere
        raise ImportError
    else:
        # Use colorama for nicer console output.
        from colorama import Fore, init
        from colorama import initialise
        # pytest resets the stream at the end - causes troubles. Since after
        # every output the stream is reset automatically we don't need this.
        initialise.atexit_done = True
        init()
except ImportError:
    class Fore(object):
        RED = ''
        GREEN = ''
        YELLOW = ''
        RESET = ''

NOTICE = object()
WARNING = object()
SPEED = object()

enable_speed = False
enable_warning = False
enable_notice = False

# callback, interface: level, str
debug_function = None
ignored_modules = ['jedi.evaluate.builtin', 'jedi.parser']
_debug_indent = -1
_start_time = time.time()

def reset_time():
    global _start_time, _debug_indent
    _start_time = time.time()
    _debug_indent = -1

def increase_indent(func):
    """Decorator for makin """
    def wrapper(*args, **kwargs):
        global _debug_indent
        _debug_indent += 1
        result = func(*args, **kwargs)
        _debug_indent -= 1
        return result
    return wrapper

def dbg(message, *args):
    """ Looks at the stack, to see if a debug message should be printed. """
    if debug_function and enable_notice:
        frm = inspect.stack()[1]
        mod = inspect.getmodule(frm[0])
        if not (mod.__name__ in ignored_modules):
            i = ' ' * _debug_indent
            debug_function(NOTICE, i + 'dbg: ' + message % args)

def warning(message, *args):
    if debug_function and enable_warning:
        i = ' ' * _debug_indent
        debug_function(WARNING, i + 'warning: ' + message % args)

def speed(name):
    if debug_function and enable_speed:
        now = time.time()
        i = ' ' * _debug_indent
        debug_function(SPEED, i + 'speed: ' + '%s %s' % (name, now - _start_time))

def print_to_stdout(level, str_out):
    """ The default debug function """
    if level == NOTICE:
        col = Fore.GREEN
    elif level == WARNING:
        col = Fore.RED
    else:
        col = Fore.YELLOW
    if not is_py3:
        str_out = str_out.encode(encoding, 'replace')
    print(col + str_out + Fore.RESET)

# debug_function = print_to_stdout
<commit_msg>small debug change<commit_after>from jedi._compatibility import encoding, is_py3
import inspect
import os
import time

try:
    if os.name == 'nt':
        # does not work on Windows, as pyreadline and colorama interfere
        raise ImportError
    else:
        # Use colorama for nicer console output.
        from colorama import Fore, init
        from colorama import initialise
        # pytest resets the stream at the end - causes troubles. Since after
        # every output the stream is reset automatically we don't need this.
        initialise.atexit_done = True
        init()
except ImportError:
    class Fore(object):
        RED = ''
        GREEN = ''
        YELLOW = ''
        RESET = ''

NOTICE = object()
WARNING = object()
SPEED = object()

enable_speed = False
enable_warning = False
enable_notice = False

# callback, interface: level, str
debug_function = None
ignored_modules = ['jedi.evaluate.builtin', 'jedi.parser']
_debug_indent = -1
_start_time = time.time()

def reset_time():
    global _start_time, _debug_indent
    _start_time = time.time()
    _debug_indent = -1

def increase_indent(func):
    """Decorator for makin """
    def wrapper(*args, **kwargs):
        global _debug_indent
        _debug_indent += 1
        try:
            result = func(*args, **kwargs)
        finally:
            _debug_indent -= 1
        return result
    return wrapper

def dbg(message, *args):
    """ Looks at the stack, to see if a debug message should be printed. """
    if debug_function and enable_notice:
        frm = inspect.stack()[1]
        mod = inspect.getmodule(frm[0])
        if not (mod.__name__ in ignored_modules):
            i = ' ' * _debug_indent
            debug_function(NOTICE, i + 'dbg: ' + message % args)

def warning(message, *args):
    if debug_function and enable_warning:
        i = ' ' * _debug_indent
        debug_function(WARNING, i + 'warning: ' + message % args)

def speed(name):
    if debug_function and enable_speed:
        now = time.time()
        i = ' ' * _debug_indent
        debug_function(SPEED, i + 'speed: ' + '%s %s' % (name, now - _start_time))

def print_to_stdout(level, str_out):
    """ The default debug function """
    if level == NOTICE:
        col = Fore.GREEN
    elif level == WARNING:
        col = Fore.RED
    else:
        col = Fore.YELLOW
    if not is_py3:
        str_out = str_out.encode(encoding, 'replace')
    print(col + str_out + Fore.RESET)

# debug_function = print_to_stdout
<|endoftext|>


<commit_before>from django.contrib import admin
from django.template import RequestContext
from django.conf.urls import patterns, url
from django.contrib import messages
from django.shortcuts import render_to_response
from django.contrib.contenttypes.models import ContentType
from django.contrib.gis.admin import OSMGeoAdmin
from django.contrib.gis.geos import GEOSGeometry

from .models import ExportConfig, ExportFormat, ExportProfile, Job, Region

admin.site.register(ExportFormat)
admin.site.register(ExportProfile)

class HOTRegionGeoAdmin(OSMGeoAdmin):
    """
    Admin model to allow Region editing in admin interface.

    Uses OSM for base layer in map in admin.
    """
    model = Region
    exclude = ['the_geom', 'the_geog']

    def save_model(self, request, obj, form, change):  # pragma no cover
        geom_merc = obj.the_geom_webmercator
        obj.the_geom = geom_merc.transform(ct=4326, clone=True)
        obj.the_geog = GEOSGeometry(obj.the_geom.wkt)
        obj.save()

class JobAdmin(OSMGeoAdmin):
    """
    Admin model for editing Jobs in the admin interface.
    """
    search_fields = ['uid', 'name', 'user__username']
    list_display = ['uid', 'name', 'user', 'region']
    exclude = ['the_geom', 'the_geog']
    actions = ['select_exports']

    update_template = 'admin/update_regions.html'
    update_complete_template = 'admin/update_complete.html'

    def select_exports(self, request, queryset):
        """
        Select exports to update.
        """
        selected = ','.join(request.POST.getlist(admin.ACTION_CHECKBOX_NAME))
        regions = Region.objects.all()

        return render_to_response(self.update_template, {
            'regions': regions,
            'selected': selected,
            'opts': self.model._meta,
        }, context_instance=RequestContext(request))

    select_exports.short_description = "Assign a region to the selected exports"

    def update_exports(self, request):
        """
        Update selected exports.
        """
        selected = request.POST.get('selected','')
        num_selected = len(selected.split(','))
        region_uid = request.POST.get('region','')
        region = Region.objects.get(uid=region_uid)
        for id in selected.split(','):
            export = Job.objects.get(id=id)
            export.region = region
            export.save()

        messages.success(request, '{0} exports updated.'.format(num_selected))
        return render_to_response(self.update_complete_template, {
            'num_selected': len(selected.split(',')),
            'region': region.name,
            'opts': self.model._meta,
        }, context_instance=RequestContext(request))

    def get_urls(self):
        urls = super(JobAdmin, self).get_urls()
        update_urls = patterns('',
            url(r'^select/$', self.admin_site.admin_view(self.select_exports)),
            url(r'^update/$', self.admin_site.admin_view(self.update_exports), name="update_regions"),
        )
        return update_urls + urls

class ExportConfigAdmin(admin.ModelAdmin):
    """
    Admin model for editing export configurations in the admin interface.
    """
    search_fields = ['uid', 'name', 'user__username']
    list_display = ['uid', 'name', 'user', 'config_type']

# register the new admin models
admin.site.register(Region, HOTRegionGeoAdmin)
admin.site.register(Job, JobAdmin)
admin.site.register(ExportConfig, ExportConfigAdmin)
<commit_msg>Fix for #86. Adding search by region name in admin.<commit_after>from django.contrib import admin
from django.template import RequestContext
from django.conf.urls import patterns, url
from django.contrib import messages
from django.shortcuts import render_to_response
from django.contrib.contenttypes.models import ContentType
from django.contrib.gis.admin import OSMGeoAdmin
from django.contrib.gis.geos import GEOSGeometry

from .models import ExportConfig, ExportFormat, ExportProfile, Job, Region

admin.site.register(ExportFormat)
admin.site.register(ExportProfile)

class HOTRegionGeoAdmin(OSMGeoAdmin):
    """
    Admin model to allow Region editing in admin interface.

    Uses OSM for base layer in map in admin.
    """
    model = Region
    exclude = ['the_geom', 'the_geog']

    def save_model(self, request, obj, form, change):  # pragma no cover
        geom_merc = obj.the_geom_webmercator
        obj.the_geom = geom_merc.transform(ct=4326, clone=True)
        obj.the_geog = GEOSGeometry(obj.the_geom.wkt)
        obj.save()

class JobAdmin(OSMGeoAdmin):
    """
    Admin model for editing Jobs in the admin interface.
    """
    search_fields = ['uid', 'name', 'user__username', 'region__name']
    list_display = ['uid', 'name', 'user', 'region']
    exclude = ['the_geom', 'the_geog']
    actions = ['select_exports']

    update_template = 'admin/update_regions.html'
    update_complete_template = 'admin/update_complete.html'

    def select_exports(self, request, queryset):
        """
        Select exports to update.
        """
        selected = ','.join(request.POST.getlist(admin.ACTION_CHECKBOX_NAME))
        regions = Region.objects.all()

        return render_to_response(self.update_template, {
            'regions': regions,
            'selected': selected,
            'opts': self.model._meta,
        }, context_instance=RequestContext(request))

    select_exports.short_description = "Assign a region to the selected exports"

    def update_exports(self, request):
        """
        Update selected exports.
        """
        selected = request.POST.get('selected','')
        num_selected = len(selected.split(','))
        region_uid = request.POST.get('region','')
        region = Region.objects.get(uid=region_uid)
        for id in selected.split(','):
            export = Job.objects.get(id=id)
            export.region = region
            export.save()

        messages.success(request, '{0} exports updated.'.format(num_selected))
        return render_to_response(self.update_complete_template, {
            'num_selected': len(selected.split(',')),
            'region': region.name,
            'opts': self.model._meta,
        }, context_instance=RequestContext(request))

    def get_urls(self):
        urls = super(JobAdmin, self).get_urls()
        update_urls = patterns('',
            url(r'^select/$', self.admin_site.admin_view(self.select_exports)),
            url(r'^update/$', self.admin_site.admin_view(self.update_exports), name="update_regions"),
        )
        return update_urls + urls

class ExportConfigAdmin(admin.ModelAdmin):
    """
    Admin model for editing export configurations in the admin interface.
    """
    search_fields = ['uid', 'name', 'user__username']
    list_display = ['uid', 'name', 'user', 'config_type']

# register the new admin models
admin.site.register(Region, HOTRegionGeoAdmin)
admin.site.register(Job, JobAdmin)
admin.site.register(ExportConfig, ExportConfigAdmin)
<|endoftext|>


<commit_before>#! /usr/bin/env python

import logging

import network
import network_agent

def random_select_and_update(network_of_agents):
    n = len(network_of_agents.G)

    # randomly select nodes from network_of_agents
    # select 10% of the nodes for update, performs floor division
    num_update = n // 10
    agents_for_update = network_of_agents.sample_network(num_update)
    print(agents_for_update)

    print(network_of_agents.G.nodes()[agents_for_update[0].get_key()])

    # update agents who were selected
    for selected_agent in agents_for_update:
        print("updating: ",
              network_of_agents.G.nodes()[selected_agent.get_key()])
        print('pre-update binary_state', selected_agent.binary_state)
        selected_agent.update_agent_binary_state()
        print('post-update_agent_binary_state', selected_agent.binary_state)

def step(time_tick, network_of_agents):
    random_select_and_update(network_of_agents)
    network_of_agents.write_network_agent_step_info(
        time_tick, '../output/network_of_agents.pout', 'a')

def main():
    # set up logging to file - see previous section for more details
    logging.basicConfig(level=logging.DEBUG,
                        format='%(asctime)s %(name)-12s %(levelname)-8s %(message)s',
                        datefmt='%m-%d %H:%M',
                        filename='myapp.log',
                        filemode='w')

    # define a Handler which writes INFO messages or higher to the sys.stderr
    console = logging.StreamHandler()
    console.setLevel(logging.INFO)

    # set a format which is simpler for console use
    formatter = logging.Formatter('%(name)-12s: %(levelname)-8s %(message)s')

    # tell the handler to use this format
    console.setFormatter(formatter)

    # add the handler to the root logger
    logging.getLogger('').addHandler(console)

    # Now, we can log to the root logger, or any other logger. First the root..
    logging.info('Logger created in main()')

    # Now, define a couple of other loggers which might represent areas in your
    # application:

    logger1 = logging.getLogger('myapp.area1')
    logger2 = logging.getLogger('myapp.area2')

    logger1.info('In main.main()')
    logger1.info('Starting Mulit Agent Neural Network (MANN)')

    # creating n number of agents
    n = 20
    logger1.debug('Number of agents to create: %s', str(n))

    # probablity for edge creation [0, 1]
    p = 0.1
    logger1.debug('Probablity for edge creation: %s', str(p))

    # Create Erdos-Renyi graph
    my_network = network.DirectedFastGNPRandomGraph(n, p)

    print("network edge list to copy\n", my_network.G.edges())  # edge list
    logger1.info('Network edge list to copy: %s', str(my_network.G.edges()))

    print(my_network.G.edges_iter())

    generated_graph_dir = '../output/mann-generated.png'
    my_network.show_graph(generated_graph_dir)
    logger1.info('Generated graph saved in %s', '../output/mann-generated.png')

    network_of_agents = network_agent.NetworkAgent()
    network_of_agents.create_multidigraph_of_agents_from_edge_list(
        n, my_network.G.edges_iter())

    network_of_agents.write_network_agent_step_info(
        -1, '../output/network_of_agents.pout', 'w')

    # make agents aware of predecessors
    # predecessors are agents who influence the current agent
    network_of_agents.set_predecessors_for_each_node()

    # randomly select nodes from network_of_agents to seed
    num_seed = 5
    agents_to_seed = network_of_agents.sample_network(num_seed)
    print("agents to seed: ", agents_to_seed)
    logger1.info('Agents seeded: %s', str(agents_to_seed))

    # seed agents who were select
    for selected_agent in agents_to_seed:
        print("seeding: ",
              network_of_agents.G.nodes()[selected_agent.get_key()])
        logger1.info('Seeding agent  %s', str(selected_agent.get_key()))

        print('pre-seed binary_state', selected_agent.binary_state)
        logger1.debug('Agent %s, pre-seed state: %s',
                      str(selected_agent.get_key()),
                      str(selected_agent.binary_state))

        selected_agent.set_binary_state(1)
        logger1.debug('Agent %s seeded', str(selected_agent.get_key()))

        print('post-seed_agent_binary_state', selected_agent.binary_state)
        logger1.debug('Agent %s, post-seed state: %s',
                      str(selected_agent.get_key()),
                      str(selected_agent.binary_state))

    logger1.info('Begin steps')
    for i in range(5):
        print("STEP # ", i)
        step(i, network_of_agents)

if __name__ == "__main__":
    print("Running")
    main()
<commit_msg>comment out print statements (logger will display non DEBUG)<commit_after>#! /usr/bin/env python

import logging

import network
import network_agent

def random_select_and_update(network_of_agents):
    n = len(network_of_agents.G)

    # randomly select nodes from network_of_agents
    # select 10% of the nodes for update, performs floor division
    num_update = n // 10
    agents_for_update = network_of_agents.sample_network(num_update)
    print(agents_for_update)

    print(network_of_agents.G.nodes()[agents_for_update[0].get_key()])

    # update agents who were selected
    for selected_agent in agents_for_update:
        print("updating: ",
              network_of_agents.G.nodes()[selected_agent.get_key()])
        print('pre-update binary_state', selected_agent.binary_state)
        selected_agent.update_agent_binary_state()
        print('post-update_agent_binary_state', selected_agent.binary_state)

def step(time_tick, network_of_agents):
    random_select_and_update(network_of_agents)
    network_of_agents.write_network_agent_step_info(
        time_tick, '../output/network_of_agents.pout', 'a')

def main():
    # set up logging to file - see previous section for more details
    logging.basicConfig(level=logging.DEBUG,
                        format='%(asctime)s %(name)-12s %(levelname)-8s %(message)s',
                        datefmt='%m-%d %H:%M',
                        filename='myapp.log',
                        filemode='w')

    # define a Handler which writes INFO messages or higher to the sys.stderr
    console = logging.StreamHandler()
    console.setLevel(logging.INFO)

    # set a format which is simpler for console use
    formatter = logging.Formatter('%(name)-12s: %(levelname)-8s %(message)s')

    # tell the handler to use this format
    console.setFormatter(formatter)

    # add the handler to the root logger
    logging.getLogger('').addHandler(console)

    # Now, we can log to the root logger, or any other logger. First the root..
    logging.info('Logger created in main()')

    # Now, define a couple of other loggers which might represent areas in your
    # application:

    logger1 = logging.getLogger('myapp.area1')
    logger2 = logging.getLogger('myapp.area2')

    logger1.info('In main.main()')
    logger1.info('Starting Mulit Agent Neural Network (MANN)')

    # creating n number of agents
    n = 20
    logger1.debug('Number of agents to create: %s', str(n))

    # probablity for edge creation [0, 1]
    p = 0.1
    logger1.debug('Probablity for edge creation: %s', str(p))

    # Create Erdos-Renyi graph
    my_network = network.DirectedFastGNPRandomGraph(n, p)

    # print("network edge list to copy\n", my_network.G.edges())  # edge list
    logger1.info('Network edge list to copy: %s', str(my_network.G.edges()))

    # print(my_network.G.edges_iter())

    generated_graph_dir = '../output/mann-generated.png'
    my_network.show_graph(generated_graph_dir)
    logger1.info('Generated graph saved in %s', '../output/mann-generated.png')

    network_of_agents = network_agent.NetworkAgent()
    network_of_agents.create_multidigraph_of_agents_from_edge_list(
        n, my_network.G.edges_iter())

    network_of_agents.write_network_agent_step_info(
        -1, '../output/network_of_agents.pout', 'w')

    # make agents aware of predecessors
    # predecessors are agents who influence the current agent
    network_of_agents.set_predecessors_for_each_node()

    # randomly select nodes from network_of_agents to seed
    num_seed = 5
    agents_to_seed = network_of_agents.sample_network(num_seed)
    # print("agents to seed: ", agents_to_seed)
    logger1.info('Agents seeded: %s', str(agents_to_seed))

    # seed agents who were select
    for selected_agent in agents_to_seed:
        # print("seeding: ",
        #       network_of_agents.G.nodes()[selected_agent.get_key()])
        logger1.info('Seeding agent  %s', str(selected_agent.get_key()))

        # print('pre-seed binary_state', selected_agent.binary_state)
        logger1.debug('Agent %s, pre-seed state: %s',
                      str(selected_agent.get_key()),
                      str(selected_agent.binary_state))

        selected_agent.set_binary_state(1)
        logger1.debug('Agent %s seeded', str(selected_agent.get_key()))

        # print('post-seed_agent_binary_state', selected_agent.binary_state)
        logger1.debug('Agent %s, post-seed state: %s',
                      str(selected_agent.get_key()),
                      str(selected_agent.binary_state))

    logger1.info('Begin steps')
    for i in range(5):
        print("STEP # ", i)
        step(i, network_of_agents)

if __name__ == "__main__":
    print("Running")
    main()
<|endoftext|>


<commit_before>#Standard imports
import unittest
import json
import logging
import numpy as np
from datetime import datetime, timedelta

# Our imports
from emission.core.get_database import get_db, get_mode_db, get_section_db
import emission.analysis.classification.inference.mode as pipeline
from emission.core.wrapper.user import User
from emission.core.wrapper.client import Client
import emission.tests.common

logging.basicConfig(level=logging.DEBUG)

class TestPipeline(unittest.TestCase):
  def setUp(self):
    self.testUsers = ["test@example.com", "best@example.com", "fest@example.com",
                      "rest@example.com", "nest@example.com"]
    self.serverName = 'localhost'

    # Sometimes, we may have entries left behind in the database if one of the tests failed
    # or threw an exception, so let us start by cleaning up all entries
    emission.tests.common.dropAllCollections(get_db())

    self.ModesColl = get_mode_db()
    self.assertEquals(self.ModesColl.find().count(), 0)

    self.SectionsColl = get_section_db()
    self.assertEquals(self.SectionsColl.find().count(), 0)

    emission.tests.common.loadTable(self.serverName, "Stage_Modes", "emission/tests/data/modes.json")
    emission.tests.common.loadTable(self.serverName, "Stage_Sections", "emission/tests/data/testModeInferFile")

    # Let's make sure that the users are registered so that they have profiles
    for userEmail in self.testUsers:
      User.register(userEmail)

    self.now = datetime.now()
    self.dayago = self.now - timedelta(days=1)
    self.weekago = self.now - timedelta(weeks = 1)

    for section in self.SectionsColl.find():
      section['section_start_datetime'] = self.dayago
      section['section_end_datetime'] = self.dayago + timedelta(hours = 1)
      if (section['confirmed_mode'] == 5):
        # We only cluster bus and train trips
        # And our test data only has bus trips
        section['section_start_point'] = {u'type': u'Point', u'coordinates': [-122.270039042, 37.8800285728]}
        section['section_end_point'] = {u'type': u'Point', u'coordinates': [-122.2690412952, 37.8739578595]}
      # print("Section start = %s, section end = %s" %
      #   (section['section_start_datetime'], section['section_end_datetime']))
      # Replace the user email with the UUID
      section['user_id'] = User.fromEmail(section['user_id']).uuid
      self.SectionsColl.save(section)

    self.pipeline = pipeline.ModeInferencePipeline()
    self.testLoadTrainingData()

  def tearDown(self):
    for testUser in self.testUsers:
      emission.tests.common.purgeSectionData(self.SectionsColl, testUser)
    logging.debug("Number of sections after purge is %d" % self.SectionsColl.find().count())
    self.ModesColl.remove()
    self.assertEquals(self.ModesColl.find().count(), 0)

  def testLoadTrainingData(self):
    allConfirmedTripsQuery = pipeline.ModeInferencePipeline.getSectionQueryWithGroundTruth({'$ne': ''})
    (self.pipeline.modeList, self.pipeline.confirmedSections) = self.pipeline.loadTrainingDataStep(allConfirmedTripsQuery)
    self.assertEquals(self.pipeline.confirmedSections.count(), len(self.testUsers) * 2)

  def testGenerateBusAndTrainStops(self):
    (self.pipeline.bus_cluster, self.pipeline.train_cluster) = self.pipeline.generateBusAndTrainStopStep()
    # Half our trips are bus, and are copies of the identical bus trip.
    # So they should all cluster into one set of start and stop points.
    # So we expect to have to cluster points - one for start and one for end
    self.assertEquals(len(self.pipeline.train_cluster), 0)
    self.assertEquals(len(self.pipeline.bus_cluster), 2)

  def testFeatureGenWithOnePoint(self):
    trackpoint1 = {"track_location": {"coordinates": [-122.0861645, 37.3910201]},
                   "time" : "20150127T203305-0800"}
    now = datetime.now()

    # ensure that the start and end datetimes are the same, since the average calculation uses
    # the total distance and the total duration
    testSeg = {"track_points": [trackpoint1],
               "distance": 500,
               "section_start_datetime": now,
               "section_end_datetime": now,
               "mode": 1,
               "section_id": 2}

    featureMatrix = np.zeros([1, len(self.pipeline.featureLabels)])
    resultVector = np.zeros(1)
    self.pipeline.updateFeatureMatrixRowWithSection(featureMatrix, 0, testSeg)
    self.assertEqual(np.count_nonzero(featureMatrix[0][4:16]), 0)
    self.assertEqual(np.count_nonzero(featureMatrix[0][19:21]), 0)

  def testGenerateTrainingSet(self):
    self.testLoadTrainingData()
    self.testGenerateBusAndTrainStops()

    (self.pipeline.featureMatrix, self.pipeline.resultVector) = self.pipeline.generateFeatureMatrixAndResultVectorStep()
    print "Number of sections = %s" % self.pipeline.confirmedSections.count()
    print "Feature Matrix shape = %s" % str(self.pipeline.featureMatrix.shape)
    self.assertEquals(self.pipeline.featureMatrix.shape[0], self.pipeline.confirmedSections.count())
    self.assertEquals(self.pipeline.featureMatrix.shape[1], len(self.pipeline.featureLabels))

  def testCleanDataStep(self):
    # Add in some entries that should be cleaned by duplicating existing sections
    runSec = self.SectionsColl.find_one({'type':'move'})
    runSec['_id'] = 'clean_me_1'
    runSec['confirmed_mode'] = 2
    logging.debug("Inserting runSec %s" % runSec)
    self.SectionsColl.insert(runSec)

    # Outlier trip
    longTripSec = self.SectionsColl.find_one({'type':'move'})
    longTripSec['_id'] = 'clean_me_2'
    longTripSec['distance'] = 5000000
    logging.debug("Inserting longTripSec %s" % longTripSec)
    self.SectionsColl.insert(longTripSec)

    unknownTripSec = self.SectionsColl.find_one({'type':'move'})
    unknownTripSec['_id'] = 'clean_me_3'
    unknownTripSec['mode'] = 'airplane'
    logging.debug("Inserting unknownTripSec %s" % unknownTripSec)
    self.SectionsColl.insert(unknownTripSec)

    allConfirmedTripsQuery = {"$and": [{'type': 'move'}, {'confirmed_mode': {'$ne': ''}}]}
    (self.pipeline.modeList, self.pipeline.confirmedSections) = self.pipeline.loadTrainingDataStep(allConfirmedTripsQuery)
    self.testGenerateBusAndTrainStops()
    (self.pipeline.featureMatrix, self.pipeline.resultVector) = self.pipeline.generateFeatureMatrixAndResultVectorStep()

    (self.pipeline.cleanedFeatureMatrix, self.pipeline.cleanedResultVector) = self.pipeline.cleanDataStep()
    self.assertEquals(self.pipeline.cleanedFeatureMatrix.shape[0], self.pipeline.confirmedSections.count() - 2)

  def testSelectFeatureIndicesStep(self):
    self.testCleanDataStep()

    self.pipeline.selFeatureIndices = self.pipeline.selectFeatureIndicesStep()
    self.assertEqual(len(self.pipeline.selFeatureIndices), 13)
    self.pipeline.selFeatureMatrix = self.pipeline.cleanedFeatureMatrix[:,self.pipeline.selFeatureIndices]
    self.assertEqual(self.pipeline.selFeatureMatrix.shape[1], len(self.pipeline.selFeatureIndices))

  def testBuildModelStep(self):
    self.testSelectFeatureIndicesStep()

    self.pipeline.model = self.pipeline.buildModelStep()
    from sklearn import cross_validation
    scores = cross_validation.cross_val_score(self.pipeline.model, self.pipeline.cleanedFeatureMatrix, self.pipeline.cleanedResultVector, cv=3)
    self.assertGreater(scores.mean(), 0.90)

  def setupTestTrips(self):
    # Generate some test data by taking existing training data and stripping out the labels
    test_id_1 = self.SectionsColl.find_one({'confirmed_mode':1})
    test_id_1['_id'] = 'test_id_1'
    test_id_1['confirmed_mode'] = ''
    logging.debug("Inserting test_id_1 %s" % test_id_1)
    self.SectionsColl.insert(test_id_1)

    test_id_2 = self.SectionsColl.find_one({'confirmed_mode':5})
    test_id_2['_id'] = 'test_id_2'
    test_id_2['confirmed_mode'] = ''
    logging.debug("Inserting test_id_2 %s" % test_id_2)
    self.SectionsColl.insert(test_id_2)

  def testGenerateFeatureMatrixAndIds(self):
    self.setupTestTrips()
    self.testBuildModelStep()
    toPredictTripsQuery = {"$and": [{'type': 'move'}, {'confirmed_mode': ''},
    # TODO: Change to does not exist
      {'predicted_mode': None}]}
    (self.pipeline.toPredictFeatureMatrix, self.pipeline.sectionIds, self.pipeline.sectionUserIds) = self.pipeline.generateFeatureMatrixAndIDsStep(toPredictTripsQuery)
    self.assertEqual(self.pipeline.toPredictFeatureMatrix.shape[0], len(self.pipeline.sectionIds))
    self.assertEqual(self.pipeline.toPredictFeatureMatrix.shape[0], len(self.pipeline.sectionUserIds))
    self.assertEqual(self.pipeline.toPredictFeatureMatrix.shape[1], len(self.pipeline.selFeatureIndices))
    self.assertEqual(self.pipeline.toPredictFeatureMatrix.shape[0], 2)

  def testPredictedProb(self):
    self.testGenerateFeatureMatrixAndIds()

    self.pipeline.predictedProb = self.pipeline.predictModesStep()
    self.assertEqual(self.pipeline.predictedProb.shape[0], 2)
    self.assertEqual(self.pipeline.predictedProb.shape[1], len(set(self.pipeline.cleanedResultVector)))

    # I know this from looking at the output for this small dataset
    # self.assertAlmostEqual(self.pipeline.predictedProb[0,0], 0.9, places=3)
    self.assertEqual(self.pipeline.predictedProb[0,0], 1)
    logging.debug("predicted prob = %s" % self.pipeline.predictedProb)
    self.assertTrue(round(self.pipeline.predictedProb[0,1],2) == 0 or
        round(self.pipeline.predictedProb[0,1],2) == 0.1)
    self.assertTrue(round(self.pipeline.predictedProb[1,0],2) == 0 or
        round(self.pipeline.predictedProb[0,1],2) == 0.1)
    self.assertEqual(self.pipeline.predictedProb[1,1], 1)

  def testConvertPredictedProbToMap(self):
    self.testPredictedProb()

    uniqueModes = sorted(set(self.pipeline.cleanedResultVector))
    self.assertEquals(uniqueModes, [1,5])

    currProb = self.pipeline.convertPredictedProbToMap(self.pipeline.modeList,
      uniqueModes, self.pipeline.predictedProb[1])

    self.assertEquals(currProb, {'bus': 1})

  def testSavePredictionsStep(self):
    self.testPredictedProb()
    self.pipeline.savePredictionsStep()
    # Confirm that the predictions are saved correctly

    test_id_1_sec = self.SectionsColl.find_one({'_id': 'test_id_1'})
    self.assertIsNotNone(test_id_1_sec['predicted_mode'])
    self.assertEquals(test_id_1_sec['predicted_mode'], {'walking': 1})

    test_id_2_sec = self.SectionsColl.find_one({'_id': 'test_id_2'})
    self.assertIsNotNone(test_id_2_sec['predicted_mode'])
    self.assertEquals(test_id_2_sec['predicted_mode'], {'bus': 1})

    # Let's make sure that we didn't accidentally mess up other fields
    self.assertIsNotNone(test_id_1_sec['distance'])
    self.assertIsNotNone(test_id_2_sec['trip_id'])

  def testSavePredictionsStepWithClient(self):
    from emission.core.wrapper.user import User

    fakeEmail = "fest@example.com"

    client = Client("testclient")
    client.update(createKey = False)
    emission.tests.common.makeValid(client)

    (resultPre, resultReg) = client.preRegister("this_is_the_super_secret_id", fakeEmail)
    self.assertEqual(resultPre, 0)
    self.assertEqual(resultReg, 1)

    user = User.fromEmail(fakeEmail)
    self.assertEqual(user.getFirstStudy(), 'testclient')

    self.testPredictedProb()
    self.pipeline.savePredictionsStep()
    # Confirm that the predictions are saved correctly

    test_id_1_sec = self.SectionsColl.find_one({'_id': 'test_id_1'})
    self.assertIsNotNone(test_id_1_sec['predicted_mode'])
    self.assertEquals(test_id_1_sec['predicted_mode'], {'walking': 1})
    self.assertEquals(test_id_1_sec['test_auto_confirmed'], {'mode': 1, 'prob': 1.0})

    test_id_2_sec = self.SectionsColl.find_one({'_id': 'test_id_2'})
    self.assertIsNotNone(test_id_2_sec['predicted_mode'])
    self.assertEquals(test_id_2_sec['predicted_mode'], {'bus': 1})
    self.assertEquals(test_id_2_sec['test_auto_confirmed'], {'mode': 5, 'prob': 1.0})

    # Let's make sure that we didn't accidentally mess up other fields
    self.assertIsNotNone(test_id_1_sec['distance'])
    self.assertIsNotNone(test_id_2_sec['trip_id'])

  def testEntirePipeline(self):
    self.setupTestTrips()
    # Here, we only have 5 trips, so the pipeline looks for the backup training
    # set instead, which fails because there is no backup. So let's copy data from
    # the main DB to the backup DB to make this test pass
    from pymongo import MongoClient
    MongoClient('localhost').drop_database("Backup_database")
    MongoClient('localhost').copy_database("Stage_database","Backup_database","localhost")
    self.pipeline.runPipeline()

    # Checks are largely the same as above
    test_id_1_sec = self.SectionsColl.find_one({'_id': 'test_id_1'})
    self.assertIsNotNone(test_id_1_sec['predicted_mode'])
    self.assertEquals(test_id_1_sec['predicted_mode'], {'walking': 1})
    self.assertIsNotNone(test_id_1_sec['distance'])

if __name__ == '__main__':
    unittest.main()
<commit_msg>Add additional logging for intermittent failures in the mode inference pipeline<commit_after>#Standard imports
import unittest
import json
import logging
import numpy as np
from datetime import datetime, timedelta

# Our imports
from emission.core.get_database import get_db, get_mode_db, get_section_db
import emission.analysis.classification.inference.mode as pipeline
from emission.core.wrapper.user import User
from emission.core.wrapper.client import Client
import emission.tests.common

logging.basicConfig(level=logging.DEBUG)

class TestPipeline(unittest.TestCase):
  def setUp(self):
    self.testUsers = ["test@example.com", "best@example.com", "fest@example.com",
                      "rest@example.com", "nest@example.com"]
    self.serverName = 'localhost'

    # Sometimes, we may have entries left behind in the database if one of the tests failed
    # or threw an exception, so let us start by cleaning up all entries
    emission.tests.common.dropAllCollections(get_db())

    self.ModesColl = get_mode_db()
    self.assertEquals(self.ModesColl.find().count(), 0)

    self.SectionsColl = get_section_db()
    self.assertEquals(self.SectionsColl.find().count(), 0)

    emission.tests.common.loadTable(self.serverName, "Stage_Modes", "emission/tests/data/modes.json")
    emission.tests.common.loadTable(self.serverName, "Stage_Sections", "emission/tests/data/testModeInferFile")

    # Let's make sure that the users are registered so that they have profiles
    for userEmail in self.testUsers:
      User.register(userEmail)

    self.now = datetime.now()
    self.dayago = self.now - timedelta(days=1)
    self.weekago = self.now - timedelta(weeks = 1)

    for section in self.SectionsColl.find():
      section['section_start_datetime'] = self.dayago
      section['section_end_datetime'] = self.dayago + timedelta(hours = 1)
      if (section['confirmed_mode'] == 5):
        # We only cluster bus and train trips
        # And our test data only has bus trips
        section['section_start_point'] = {u'type': u'Point', u'coordinates': [-122.270039042, 37.8800285728]}
        section['section_end_point'] = {u'type': u'Point', u'coordinates': [-122.2690412952, 37.8739578595]}
      # print("Section start = %s, section end = %s" %
      #   (section['section_start_datetime'], section['section_end_datetime']))
      # Replace the user email with the UUID
      section['user_id'] = User.fromEmail(section['user_id']).uuid
      self.SectionsColl.save(section)

    self.pipeline = pipeline.ModeInferencePipeline()
    self.testLoadTrainingData()

  def tearDown(self):
    for testUser in self.testUsers:
      emission.tests.common.purgeSectionData(self.SectionsColl, testUser)
    logging.debug("Number of sections after purge is %d" % self.SectionsColl.find().count())
    self.ModesColl.remove()
    self.assertEquals(self.ModesColl.find().count(), 0)

  def testLoadTrainingData(self):
    allConfirmedTripsQuery = pipeline.ModeInferencePipeline.getSectionQueryWithGroundTruth({'$ne': ''})
    (self.pipeline.modeList, self.pipeline.confirmedSections) = self.pipeline.loadTrainingDataStep(allConfirmedTripsQuery)
    self.assertEquals(self.pipeline.confirmedSections.count(), len(self.testUsers) * 2)

  def testGenerateBusAndTrainStops(self):
    (self.pipeline.bus_cluster, self.pipeline.train_cluster) = self.pipeline.generateBusAndTrainStopStep()
    # Half our trips are bus, and are copies of the identical bus trip.
    # So they should all cluster into one set of start and stop points.
    # So we expect to have to cluster points - one for start and one for end
    self.assertEquals(len(self.pipeline.train_cluster), 0)
    self.assertEquals(len(self.pipeline.bus_cluster), 2)

  def testFeatureGenWithOnePoint(self):
    trackpoint1 = {"track_location": {"coordinates": [-122.0861645, 37.3910201]},
                   "time" : "20150127T203305-0800"}
    now = datetime.now()

    # ensure that the start and end datetimes are the same, since the average calculation uses
    # the total distance and the total duration
    testSeg = {"track_points": [trackpoint1],
               "distance": 500,
               "section_start_datetime": now,
               "section_end_datetime": now,
               "mode": 1,
               "section_id": 2}

    featureMatrix = np.zeros([1, len(self.pipeline.featureLabels)])
    resultVector = np.zeros(1)
    self.pipeline.updateFeatureMatrixRowWithSection(featureMatrix, 0, testSeg)
    self.assertEqual(np.count_nonzero(featureMatrix[0][4:16]), 0)
    self.assertEqual(np.count_nonzero(featureMatrix[0][19:21]), 0)

  def testGenerateTrainingSet(self):
    self.testLoadTrainingData()
    self.testGenerateBusAndTrainStops()

    (self.pipeline.featureMatrix, self.pipeline.resultVector) = self.pipeline.generateFeatureMatrixAndResultVectorStep()
    print "Number of sections = %s" % self.pipeline.confirmedSections.count()
    print "Feature Matrix shape = %s" % str(self.pipeline.featureMatrix.shape)
    self.assertEquals(self.pipeline.featureMatrix.shape[0], self.pipeline.confirmedSections.count())
    self.assertEquals(self.pipeline.featureMatrix.shape[1], len(self.pipeline.featureLabels))

  def testCleanDataStep(self):
    # Add in some entries that should be cleaned by duplicating existing sections
    runSec = self.SectionsColl.find_one({'type':'move'})
    runSec['_id'] = 'clean_me_1'
    runSec['confirmed_mode'] = 2
    logging.debug("Inserting runSec %s" % runSec)
    self.SectionsColl.insert(runSec)

    # Outlier trip
    longTripSec = self.SectionsColl.find_one({'type':'move'})
    longTripSec['_id'] = 'clean_me_2'
    longTripSec['distance'] = 5000000
    logging.debug("Inserting longTripSec %s" % longTripSec)
    self.SectionsColl.insert(longTripSec)

    unknownTripSec = self.SectionsColl.find_one({'type':'move'})
    unknownTripSec['_id'] = 'clean_me_3'
    unknownTripSec['mode'] = 'airplane'
    logging.debug("Inserting unknownTripSec %s" % unknownTripSec)
    self.SectionsColl.insert(unknownTripSec)

    allConfirmedTripsQuery = {"$and": [{'type': 'move'}, {'confirmed_mode': {'$ne': ''}}]}
    (self.pipeline.modeList, self.pipeline.confirmedSections) = self.pipeline.loadTrainingDataStep(allConfirmedTripsQuery)
    self.testGenerateBusAndTrainStops()
    (self.pipeline.featureMatrix, self.pipeline.resultVector) = self.pipeline.generateFeatureMatrixAndResultVectorStep()

    (self.pipeline.cleanedFeatureMatrix, self.pipeline.cleanedResultVector) = self.pipeline.cleanDataStep()
    self.assertEquals(self.pipeline.cleanedFeatureMatrix.shape[0], self.pipeline.confirmedSections.count() - 2)

  def testSelectFeatureIndicesStep(self):
    self.testCleanDataStep()

    self.pipeline.selFeatureIndices = self.pipeline.selectFeatureIndicesStep()
    self.assertEqual(len(self.pipeline.selFeatureIndices), 13)
    self.pipeline.selFeatureMatrix = self.pipeline.cleanedFeatureMatrix[:,self.pipeline.selFeatureIndices]
    self.assertEqual(self.pipeline.selFeatureMatrix.shape[1], len(self.pipeline.selFeatureIndices))

  def testBuildModelStep(self):
    self.testSelectFeatureIndicesStep()

    self.pipeline.model = self.pipeline.buildModelStep()
    from sklearn import cross_validation
    scores = cross_validation.cross_val_score(self.pipeline.model, self.pipeline.cleanedFeatureMatrix, self.pipeline.cleanedResultVector, cv=3)
    self.assertGreater(scores.mean(), 0.90)

  def setupTestTrips(self):
    # Generate some test data by taking existing training data and stripping out the labels
    test_id_1 = self.SectionsColl.find_one({'confirmed_mode':1})
    test_id_1['_id'] = 'test_id_1'
    test_id_1['confirmed_mode'] = ''
    logging.debug("Inserting test_id_1 %s" % test_id_1)
    self.SectionsColl.insert(test_id_1)

    test_id_2 = self.SectionsColl.find_one({'confirmed_mode':5})
    test_id_2['_id'] = 'test_id_2'
    test_id_2['confirmed_mode'] = ''
    logging.debug("Inserting test_id_2 %s" % test_id_2)
    self.SectionsColl.insert(test_id_2)

  def testGenerateFeatureMatrixAndIds(self):
    self.setupTestTrips()
    self.testBuildModelStep()
    toPredictTripsQuery = {"$and": [{'type': 'move'}, {'confirmed_mode': ''},
    # TODO: Change to does not exist
      {'predicted_mode': None}]}
    (self.pipeline.toPredictFeatureMatrix, self.pipeline.sectionIds, self.pipeline.sectionUserIds) = self.pipeline.generateFeatureMatrixAndIDsStep(toPredictTripsQuery)
    self.assertEqual(self.pipeline.toPredictFeatureMatrix.shape[0], len(self.pipeline.sectionIds))
    self.assertEqual(self.pipeline.toPredictFeatureMatrix.shape[0], len(self.pipeline.sectionUserIds))
    self.assertEqual(self.pipeline.toPredictFeatureMatrix.shape[1], len(self.pipeline.selFeatureIndices))
    self.assertEqual(self.pipeline.toPredictFeatureMatrix.shape[0], 2)

  def testPredictedProb(self):
    self.testGenerateFeatureMatrixAndIds()

    self.pipeline.predictedProb = self.pipeline.predictModesStep()
    self.assertEqual(self.pipeline.predictedProb.shape[0], 2)
    self.assertEqual(self.pipeline.predictedProb.shape[1], len(set(self.pipeline.cleanedResultVector)))

    # I know this from looking at the output for this small dataset
    # self.assertAlmostEqual(self.pipeline.predictedProb[0,0], 0.9, places=3)
    self.assertEqual(self.pipeline.predictedProb[0,0], 1)
    logging.debug("predicted prob = %s" % self.pipeline.predictedProb)
    self.assertTrue(round(self.pipeline.predictedProb[0,1],2) == 0 or
        round(self.pipeline.predictedProb[0,1],2) == 0.1,
        "predictedProb[0,1] = %s, with rounding = %s" % (self.pipeline.predictedProb[0,1],
            round(self.pipeline.predictedProb[0,1])))
    self.assertTrue(round(self.pipeline.predictedProb[1,0],2) == 0 or
        round(self.pipeline.predictedProb[1,0],2) == 0.1,
        "predictedProb[1,0] = %s, with rounding = %s" % (self.pipeline.predictedProb[1,0],
            round(self.pipeline.predictedProb[1,0])))
    self.assertEqual(self.pipeline.predictedProb[1,1], 1)

  def testConvertPredictedProbToMap(self):
    self.testPredictedProb()

    uniqueModes = sorted(set(self.pipeline.cleanedResultVector))
    self.assertEquals(uniqueModes, [1,5])

    currProb = self.pipeline.convertPredictedProbToMap(self.pipeline.modeList,
      uniqueModes, self.pipeline.predictedProb[1])

    self.assertEquals(currProb, {'bus': 1})

  def testSavePredictionsStep(self):
    self.testPredictedProb()
    self.pipeline.savePredictionsStep()
    # Confirm that the predictions are saved correctly

    test_id_1_sec = self.SectionsColl.find_one({'_id': 'test_id_1'})
    self.assertIsNotNone(test_id_1_sec['predicted_mode'])
    self.assertEquals(test_id_1_sec['predicted_mode'], {'walking': 1})

    test_id_2_sec = self.SectionsColl.find_one({'_id': 'test_id_2'})
    self.assertIsNotNone(test_id_2_sec['predicted_mode'])
    self.assertEquals(test_id_2_sec['predicted_mode'], {'bus': 1})

    # Let's make sure that we didn't accidentally mess up other fields
    self.assertIsNotNone(test_id_1_sec['distance'])
    self.assertIsNotNone(test_id_2_sec['trip_id'])

  def testSavePredictionsStepWithClient(self):
    from emission.core.wrapper.user import User

    fakeEmail = "fest@example.com"

    client = Client("testclient")
    client.update(createKey = False)
    emission.tests.common.makeValid(client)

    (resultPre, resultReg) = client.preRegister("this_is_the_super_secret_id", fakeEmail)
    self.assertEqual(resultPre, 0)
    self.assertEqual(resultReg, 1)

    user = User.fromEmail(fakeEmail)
    self.assertEqual(user.getFirstStudy(), 'testclient')

    self.testPredictedProb()
    self.pipeline.savePredictionsStep()
    # Confirm that the predictions are saved correctly

    test_id_1_sec = self.SectionsColl.find_one({'_id': 'test_id_1'})
    self.assertIsNotNone(test_id_1_sec['predicted_mode'])
    self.assertEquals(test_id_1_sec['predicted_mode'], {'walking': 1})
    self.assertEquals(test_id_1_sec['test_auto_confirmed'], {'mode': 1, 'prob': 1.0})

    test_id_2_sec = self.SectionsColl.find_one({'_id': 'test_id_2'})
    self.assertIsNotNone(test_id_2_sec['predicted_mode'])
    self.assertEquals(test_id_2_sec['predicted_mode'], {'bus': 1})
    self.assertEquals(test_id_2_sec['test_auto_confirmed'], {'mode': 5, 'prob': 1.0})

    # Let's make sure that we didn't accidentally mess up other fields
    self.assertIsNotNone(test_id_1_sec['distance'])
    self.assertIsNotNone(test_id_2_sec['trip_id'])

  def testEntirePipeline(self):
    self.setupTestTrips()
    # Here, we only have 5 trips, so the pipeline looks for the backup training
    # set instead, which fails because there is no backup. So let's copy data from
    # the main DB to the backup DB to make this test pass
    from pymongo import MongoClient
    MongoClient('localhost').drop_database("Backup_database")
    MongoClient('localhost').copy_database("Stage_database","Backup_database","localhost")
    self.pipeline.runPipeline()

    # Checks are largely the same as above
    test_id_1_sec = self.SectionsColl.find_one({'_id': 'test_id_1'})
    self.assertIsNotNone(test_id_1_sec['predicted_mode'])
    self.assertEquals(test_id_1_sec['predicted_mode'], {'walking': 1})
    self.assertIsNotNone(test_id_1_sec['distance'])

if __name__ == '__main__':
    unittest.main()
<|endoftext|>


<commit_before>import os
import unittest

import sys

import PyFBA

media_file_loc = ''
if os.path.exists('../media/ArgonneLB.txt'):
    media_file_loc = '../media'
elif os.path.exists('media/ArgonneLB.txt'):
    media_file_loc = 'media'
elif 'PYFBA_MEDIA_DIR' in os.environ and os.path.exists(os.path.join(os.environ['PYFBA_MEDIA_DIR'], 'ArgonneLB.txt')):
    media_file_loc = os.environ['PYFBA_MEDIA_DIR']
else:
    sys.stderr.write("No media found. Can't proceed with the tests.\n")
    sys.stderr.write("You can specify the media location by setting the PYFBA_MEDIA_DIR environment variable\n")

class TestReadMedia(unittest.TestCase):
    def setUp(self):
        self.assertTrue(os.path.exists(os.path.join(media_file_loc, 'ArgonneLB.txt')))

    def test_read_media_file(self):
        """Test reading a media file"""
        if media_file_loc == "":
            return
        media = PyFBA.parse.read_media_file(os.path.join(media_file_loc, 'ArgonneLB.txt'))
        self.assertEqual(len(media), 65)
        gluc = PyFBA.metabolism.CompoundWithLocation('cpd00027', 'D-Glucose', 'e')
        self.assertIn(gluc, media)

if __name__ == '__main__':
    unittest.main()
<commit_msg>Fixing testing media<commit_after>import os
import unittest

import sys

import PyFBA

media_file_loc = ''
if os.path.exists('../media/ArgonneLB.txt'):
    media_file_loc = '../media'
elif os.path.exists('media/ArgonneLB.txt'):
    media_file_loc = 'media'
elif 'PYFBA_MEDIA_DIR' in os.environ and os.path.exists(os.path.join(os.environ['PYFBA_MEDIA_DIR'], 'ArgonneLB.txt')):
    media_file_loc = os.environ['PYFBA_MEDIA_DIR']
else:
    sys.stderr.write("No media found. Can't proceed with the tests.\n")
    sys.stderr.write("You can specify the media location by setting the PYFBA_MEDIA_DIR environment variable\n")

class TestReadMedia(unittest.TestCase):
    def setUp(self):
        self.assertTrue(os.path.exists(os.path.join(media_file_loc, 'ArgonneLB.txt')))

    def test_read_media_file(self):
        """Test reading a media file"""
        if media_file_loc == "":
            return
        media = PyFBA.parse.read_media_file(os.path.join(media_file_loc, 'ArgonneLB.txt'))
        self.assertEqual(len(media), 65)
        testm = None
        for m in  media:
            if m.name == 'D-Glucose':
                testm = m
                break
        self.assertIn(testm, media)

if __name__ == '__main__':
    unittest.main()
<|endoftext|>