<commit_before>try:
    import simplejson as json
except ImportError:
    import json

from flask import Flask, Response, render_template, request
from werkzeug.contrib.cache import SimpleCache
import requests

app = Flask('roundup')
app.config.from_object('settings')
# TODO: In a production environment, you would never use an in-memory cache like
# this.  You should instead use something like Redis or Memcached.
cache = SimpleCache()

def get_top_posts(*args, **kwargs):
    kwargs['time'] = kwargs.get('time') or '6h'

    url = '{}/realtime/posts'.format(app.config['PARSELY']['api_base_url'])
    posts = []

    for apikey, secret in app.config['PARSELY']['apikeys']:
        kwargs['apikey'] = apikey
        kwargs['secret'] = secret
        apikey_posts = requests.get(url, params=kwargs).json()['data']
        for post in apikey_posts:
            post['apikey'] = apikey

        posts.extend(apikey_posts)

    posts = sorted(posts, key=lambda x: x['_hits'], reverse=True)
    return posts[:kwargs['limit']]

@app.route('/top_posts')
def top_posts():
    limit = request.args.get('limit') or app.config['NUMBER_OF_POSTS']
    if isinstance(limit, basestring):
        limit = int(limit)
    posts = cache.get('top_posts')
    if posts is None:
        posts = get_top_posts(limit=limit)
        cache.set('top_posts', posts, timeout=app.config['CACHE_EXPIRY'])

    return Response(json.dumps(posts), content_type='application/json')

@app.route('/')
def home():
    return render_template('home.jinja2.html')

if __name__ == '__main__':
    app.run()
<commit_msg>Better handling of limit param in top posts<commit_after>try:
    import simplejson as json
except ImportError:
    import json

from flask import Flask, Response, render_template, request
from werkzeug.contrib.cache import SimpleCache
import requests

app = Flask('roundup')
app.config.from_object('settings')
# TODO: In a production environment, you would never use an in-memory cache like
# this.  You should instead use something like Redis or Memcached.
cache = SimpleCache()

def get_top_posts(*args, **kwargs):
    kwargs['time'] = kwargs.get('time') or '6h'

    url = '{}/realtime/posts'.format(app.config['PARSELY']['api_base_url'])
    posts = []

    for apikey, secret in app.config['PARSELY']['apikeys']:
        kwargs['apikey'] = apikey
        kwargs['secret'] = secret
        apikey_posts = requests.get(url, params=kwargs).json()['data']
        for post in apikey_posts:
            post['apikey'] = apikey

        posts.extend(apikey_posts)

    posts = sorted(posts, key=lambda x: x['_hits'], reverse=True)
    return posts[:kwargs['limit']]

@app.route('/top_posts')
def top_posts():
    limit = request.args.get('limit', default=app.config['NUMBER_OF_POSTS'],
                             type=int)
    if limit <= 0:
        limit = app.config['NUMBER_OF_POSTS']

    posts = cache.get('top_posts')
    if posts is None:
        posts = get_top_posts(limit=limit)
        cache.set('top_posts', posts, timeout=app.config['CACHE_EXPIRY'])

    return Response(json.dumps(posts), content_type='application/json')

@app.route('/')
def home():
    return render_template('home.jinja2.html')

if __name__ == '__main__':
    app.run()
<|endoftext|>


<commit_before>import os
import sys
import datetime
import importlib
from IRCResponse import IRCResponse, ResponseType
from ModuleHandler import AutoLoadModules
from ModuleInterface import ModuleInterface
import GlobalVars

class Module(ModuleInterface):
    triggers = ["connect", "quit", "quitfrom", "restart", "shutdown"]
    help = "connect <server> <channel>, quit, quitfrom <server>, restart, shutdown - Connect to / Disconnect from servers, Restart current bot, Shut down all bots"

    def onTrigger(self, Hubbot, message):
        if message.User.Name not in GlobalVars.admins:
            return IRCResponse(ResponseType.Say, "You are not allowed to use '{}'".format(message.Command), message.ReplyTo)

        if message.Command == "connect":
            if len(message.ParameterList) >= 2:
                server_with_port = message.ParameterList[0]
                if ":" in server_with_port:
                    server = server_with_port.split(":")[0]
                    port = int(server_with_port.split(":")[1])
                else:
                    server = server_with_port
                    port = 6667
                tmpChannels = message.ParameterList[1:]
                channels = []
                for chan in tmpChannels:
                    channels.append(chan.encode("ascii", "ignore"))
                GlobalVars.bothandler.startBotFactory(server, port, channels)
                if server not in GlobalVars.bothandler.botfactories:
                    return IRCResponse(ResponseType.Say, "Could not connect to server '{}'".format(server), message.ReplyTo)

        if message.Command == "quit":
            if datetime.datetime.now() > Hubbot.startTime + datetime.timedelta(seconds=10):
                Hubbot.Quitting = True
                Hubbot.restarting = False
                quitMessage = "ohok".encode("utf-8")
                GlobalVars.bothandler.stopBotFactory(Hubbot.server, quitMessage)

        if message.Command == "quitfrom":
            if len(message.ParameterList) >= 1:
                for server in message.ParameterList:
                    if server in GlobalVars.bothandler.botfactories and server != Hubbot.server:
                        GlobalVars.bothandler.stopBotFactory(server)
                        return IRCResponse(ResponseType.Say, "Successfully quit from server '{}'".format(server), message.ReplyTo)
                    elif server == Hubbot.server:
                        return IRCResponse(ResponseType.Say, "Please use quit to quit from current server.", message.ReplyTo)
                    else:
                        return IRCResponse(ResponseType.Say, "I don't think I am on that server.", message.ReplyTo)

        if message.Command == "restart":
            if datetime.datetime.now() > Hubbot.startTime + datetime.timedelta(seconds=10):
                server = Hubbot.server
                port = GlobalVars.bothandler.botfactories[server].port
                channels = Hubbot.channels
                GlobalVars.bothandler.stopBotFactory(Hubbot.server, "Restarting...")
                del sys.modules["Hubbot"]
                os.remove("Hubbot.pyc")
                core = importlib.import_module("Hubbot")
                reload(core)
                AutoLoadModules()
                GlobalVars.bothandler.startBotFactory(server, port, channels)
                return

        if message.Command == "shutdown":
            if datetime.datetime.now() > Hubbot.startTime + datetime.timedelta(seconds=10):
                GlobalVars.bothandler.shutdown()
                return<commit_msg>Okay maybe THIS will work better.<commit_after>import os
import sys
import datetime
import importlib
from IRCResponse import IRCResponse, ResponseType
from ModuleHandler import AutoLoadModules
from ModuleInterface import ModuleInterface
import GlobalVars

class Module(ModuleInterface):
    triggers = ["connect", "quit", "quitfrom", "restart", "shutdown"]
    help = "connect <server> <channel>, quit, quitfrom <server>, restart, shutdown - Connect to / Disconnect from servers, Restart current bot, Shut down all bots"

    def onTrigger(self, Hubbot, message):
        if message.User.Name not in GlobalVars.admins:
            return IRCResponse(ResponseType.Say, "You are not allowed to use '{}'".format(message.Command), message.ReplyTo)

        if message.Command == "connect":
            if len(message.ParameterList) >= 2:
                server_with_port = message.ParameterList[0]
                if ":" in server_with_port:
                    server = server_with_port.split(":")[0]
                    port = int(server_with_port.split(":")[1])
                else:
                    server = server_with_port
                    port = 6667
                tmpChannels = message.ParameterList[1:]
                channels = []
                for chan in tmpChannels:
                    channels.append(chan.encode("ascii", "ignore"))
                GlobalVars.bothandler.startBotFactory(server, port, channels)
                if server not in GlobalVars.bothandler.botfactories:
                    return IRCResponse(ResponseType.Say, "Could not connect to server '{}'".format(server), message.ReplyTo)

        if message.Command == "quit":
            if datetime.datetime.now() > Hubbot.startTime + datetime.timedelta(seconds=10):
                Hubbot.Quitting = True
                Hubbot.restarting = False
                quitMessage = "ohok".encode("utf-8")
                GlobalVars.bothandler.stopBotFactory(Hubbot.server, quitMessage)

        if message.Command == "quitfrom":
            if len(message.ParameterList) >= 1:
                for server in message.ParameterList:
                    if server in GlobalVars.bothandler.botfactories and server != Hubbot.server:
                        GlobalVars.bothandler.stopBotFactory(server)
                        return IRCResponse(ResponseType.Say, "Successfully quit from server '{}'".format(server), message.ReplyTo)
                    elif server == Hubbot.server:
                        return IRCResponse(ResponseType.Say, "Please use quit to quit from current server.", message.ReplyTo)
                    else:
                        return IRCResponse(ResponseType.Say, "I don't think I am on that server.", message.ReplyTo)

        if message.Command == "restart":
            if datetime.datetime.now() > Hubbot.startTime + datetime.timedelta(seconds=10):
                server = Hubbot.server
                port = GlobalVars.bothandler.botfactories[server].port
                channels = Hubbot.channels
                GlobalVars.bothandler.restartBotFactory(server, port, channels)
                return

        if message.Command == "shutdown":
            if datetime.datetime.now() > Hubbot.startTime + datetime.timedelta(seconds=10):
                GlobalVars.bothandler.shutdown()
                return<|endoftext|>


<commit_before>#!/usr/bin/python
import subprocess
import signal
import os

import cherrypy
from jinja2 import Environment, FileSystemLoader

import demos

jenv = Environment(loader=FileSystemLoader('.'))

class DAVM_Launcher(object):

    myHost = ""
    fifo = None

    @cherrypy.expose
    def index(self):
        jtemp = jenv.get_template('index.html')
        return jtemp.render(omegaList=demos.omegalibDemos[DAVM_Launcher.myHost], movieList=demos.movieDemos)

    @cherrypy.expose
    def stopIt(self):
        if cherrypy.session.has_key('mySession'):
            os.killpg(cherrypy.session['mySession'].pid, signal.SIGKILL)
            # make this specific to running demo
            del cherrypy.session['mySession']
            if DAVM_Launcher.fifo != None:
                try:
                    os.close(DAVM_Launcher.fifo)
                except OSError:
                    print "stale file descriptor"

        subprocess.Popen([
            'killall', 'orun'
        ])
        raise cherrypy.HTTPRedirect('/')

    @cherrypy.expose
    def runScript(self, bino, movie):
        if movie=="":
            return self.index()

        # useful: http://bino3d.org/doc/bino.html#Script-Commands

        binoCmd = str(bino)
        print binoCmd
        if DAVM_Launcher.fifo:
            if 'rewind' in binoCmd:
                binoCmd='set-pos 0'
            if 'seekF' in binoCmd:
                binoCmd='seek 15.0'
            if 'seekB' in binoCmd:
                binoCmd='seek -15.0'
            os.write(DAVM_Launcher.fifo, binoCmd + '\n')

        jtemp = jenv.get_template('index.html')
        return jtemp.render(omegaList=demos.omegalibDemos[DAVM_Launcher.myHost], movieList=demos.movieDemos, currentDemo=movie)

    @cherrypy.expose
    def runOmega(self, button):
        args = [
            'orun',
            '-s',
            '/local/examples/' + button,
        ]
        result = subprocess.Popen(args, preexec_fn=os.setsid)
        cherrypy.session['mySession'] = result

        jtemp = jenv.get_template('index.html')
        return jtemp.render(omegaList=demos.omegalibDemos[DAVM_Launcher.myHost], movieList=demos.movieDemos, currentDemo=button)

    @cherrypy.expose
    def runMovie(self, button):
        moviePath = '/local/davm.movies/'

	pipePath = "/local/bino/fifo/bino.pipe"

        args = [
        	"/local/bino/install/bin/bino",
		"--loop",
		"-i",
		"mono",
		"--log-file=/var/tmp/bino.log",
		"-o",
		"equalizer",
		"--eq-config",
		"/etc/eq/local.auto.eqc",
		"--read-commands",
		pipePath,
		str(moviePath + button)
	]

        result = subprocess.Popen(args, preexec_fn=os.setsid)
        cherrypy.session['mySession'] = result

        DAVM_Launcher.fifo = os.open(pipePath, os.O_WRONLY)

        jtemp = jenv.get_template('index.html')
        return jtemp.render(omegaList=demos.omegalibDemos[DAVM_Launcher.myHost], movieList=demos.movieDemos, currentDemo=button)

if __name__ == '__main__':
    cherrypy.config.update({'tools.sessions.on' : True, 'server.socket_port' : 8002})
    conf = {
        '/images': {
            'tools.staticdir.on' : True,
            'tools.staticdir.dir' : '/local/examples/launcher/images'
        }
    }

    DAVM_Launcher.myHost = cherrypy.server.socket_host
    cherrypy.tree.mount(DAVM_Launcher(), "/", conf)

    cherrypy.engine.start()
    cherrypy.engine.block()
<commit_msg>Restore previous launcher behaviour<commit_after>#!/usr/bin/python
import subprocess
import signal
import os

import cherrypy
from jinja2 import Environment, FileSystemLoader

import demos

jenv = Environment(loader=FileSystemLoader('.'))

class DAVM_Launcher(object):

    myHost = ""
    fifo = None

    @cherrypy.expose
    def index(self):
        jtemp = jenv.get_template('index.html')
        return jtemp.render(omegaList=demos.omegalibDemos[DAVM_Launcher.myHost], movieList=demos.movieDemos)

    @cherrypy.expose
    def stopIt(self):
        if cherrypy.session.has_key('mySession'):
            os.killpg(cherrypy.session['mySession'].pid, signal.SIGKILL)
            # make this specific to running demo
            del cherrypy.session['mySession']
            if DAVM_Launcher.fifo != None:
                try:
                    os.close(DAVM_Launcher.fifo)
                except OSError:
                    print "stale file descriptor"

        subprocess.Popen([
            'killall', 'orun'
        ])
        raise cherrypy.HTTPRedirect('/')

    @cherrypy.expose
    def runScript(self, bino, movie):
        if movie=="":
            return self.index()

        # useful: http://bino3d.org/doc/bino.html#Script-Commands

        binoCmd = str(bino)
        print binoCmd
        if DAVM_Launcher.fifo:
            if 'rewind' in binoCmd:
                binoCmd='set-pos 0'
            if 'seekF' in binoCmd:
                binoCmd='seek 15.0'
            if 'seekB' in binoCmd:
                binoCmd='seek -15.0'
            os.write(DAVM_Launcher.fifo, binoCmd + '\n')

        jtemp = jenv.get_template('index.html')
        return jtemp.render(omegaList=demos.omegalibDemos[DAVM_Launcher.myHost], movieList=demos.movieDemos, currentDemo=movie)

    @cherrypy.expose
    def runOmega(self, button):
        args = [
            'orun',
            '-s',
            '/local/examples/' + button,
        ]
        result = subprocess.Popen(args, preexec_fn=os.setsid)
        cherrypy.session['mySession'] = result

        jtemp = jenv.get_template('index.html')
        return jtemp.render(omegaList=demos.omegalibDemos[DAVM_Launcher.myHost], movieList=demos.movieDemos, currentDemo=button)

    @cherrypy.expose
    def runMovie(self, button):
        moviePath = '/local/davm.movies/'

	pipePath = "/local/bino/fifo/bino.pipe"

        args = [
        	"/local/bino/install/bin/bino",
		"--loop",
		"-i",
		"mono",
		"--log-file=/var/tmp/bino.log",
		"-o",
		"equalizer",
		"--eq-config",
		"/etc/eq/local.auto.eqc",
		"--read-commands",
		pipePath,
		str(moviePath + button)
	]

        result = subprocess.Popen(args, preexec_fn=os.setsid)
        cherrypy.session['mySession'] = result

        DAVM_Launcher.fifo = os.open(pipePath, os.O_WRONLY)

        os.write(DAVM_Launcher.fifo, 'pause\n')
        os.write(DAVM_Launcher.fifo, 'step\n')

        jtemp = jenv.get_template('index.html')
        return jtemp.render(omegaList=demos.omegalibDemos[DAVM_Launcher.myHost], movieList=demos.movieDemos, currentDemo=button)

if __name__ == '__main__':
    cherrypy.config.update({'tools.sessions.on' : True, 'server.socket_port' : 8002})
    conf = {
        '/images': {
            'tools.staticdir.on' : True,
            'tools.staticdir.dir' : '/local/examples/launcher/images'
        }
    }

    DAVM_Launcher.myHost = cherrypy.server.socket_host
    cherrypy.tree.mount(DAVM_Launcher(), "/", conf)

    cherrypy.engine.start()
    cherrypy.engine.block()
<|endoftext|>


<commit_before>import os
import json
from fabric.api import *
from distutils.util import strtobool
from fabric.colors import green, red

class Lazyconf():

    def load_only_config(self):

        # If we're loading the config to actually serve, we can't fall back.
        self.load_data(False)

    def config(self):

        # If we're configuring, fall back on schema if the file doesn't exist.
        self.load_data(True)
        self.__cfg(self.data, 'config')
        self.save()

    def save_schema(self):
        d = self.data
        self.__sv_schema(d)

    def __sv_schema(self, d):
        for k,v in d.iteritems():
            if t is dict:
                self.__sv_schema(d)
            elif t is bool:
                v = False
            elif t is int:
                v = 0
            else:
                v = ''
        return

    def __get_list(self, k):
        if k in self.lists.keys():
            return self.lists[k]
        return None

    def __get_list_value(self, k, l):
        if k in l.keys():
            return l[k]
        return ""

    def __get_list_key(self, v, l):
        for k,val in l.iteritems():
            if v == val:
                return k
        return ""

    def __get_label(self, l):
        if l in self.labels.keys():
            return self.labels[l]
        return l

    def __cfg(self, d, key_string):

        if '_enabled' in d.keys():
            s = '.'.join([key_string,'_enabled'])
            d['_enabled'] = self.deprompt_bool(prompt(self.__get_label(s) + ' (y/n)?', default = self.prompt_bool(d['_enabled'])))
            if d['_enabled'] is False:
                return

        for k, v in d.iteritems():
            if key_string == 'config._internal':
                return

            if k == '_enabled':
                continue

            t = type(v)
            s = '.'.join([key_string,k])

            list = self.__get_list(s)
            if list:
                d[k] = self.__get_list_value(prompt(self.__get_label(s) + '(' + ','.join(list.keys()) + ')' + ':', default = self.__get_list_key(v,list)), list)
            elif t is dict:
                self.__cfg(v, s)
            elif t is str or t is unicode:
                d[k] = prompt(self.__get_label(s) + ':', default = v)
            elif t is bool:
                d[k] = self.deprompt_bool(prompt(self.__get_label(s) + ' (y/n)?', default = self.prompt_bool(v)))
            elif t is int:
                d[k] = prompt(self.__get_label(s) + ':', default = v)

    def __init__(self, project_dir, filename = 'lazyconf.json'):

        ### Project directory and filename ###
        if len(project_dir) == 0:
            project_dir = '.'

        self.project_dir = project_dir

        self.filename = filename

        self.data = {}

        self.labels = {}

        self.lists = {}

    def load_schema(self):

        # Set the path to the values from __init__.
        path = self.project_dir + '/' + self.filename

        # If we can't find a schema file in that folder, load the default schema.
        if not self.load_file(path, True):
            print(red("Could not load schema from " + path + ". Falling back to default..."))

            bkp_path = './' + self.filename

            # If we can't find the default schema, we cannot continue.
            if not self.load_file(bkp_path, True):
                print(red("Fatal: Could not load default schema from " + bkp_path + ". Aborting..."))
                exit()
            else:
                print(green("Loaded default schema."))
        else:
            print(green("Loaded schema from " + path + ".schema"))

    def load_data(self, fallback = True):

        # Set the path to the values from __init__.
        path = self.project_dir + '/' + self.filename

        # If we can't find the file in the folder and we're falling back ot schema, call load_schema.
        if not self.load_file(path):
            if fallback:
                self.load_schema()
            else:
                print(red("Fatal: Could not load schema from " + path + ". Aborting..."))
                exit()

    def load_file(self, path, schema = False):
        if schema:
            path += '.schema'
        try:
            with open(path) as handle:

                try:
                    self.data = json.load(handle)
                    self.labels = self.data['_internal']['labels']
                    self.lists = self.data['_internal']['lists']
                except ValueError as e:
                    print(red('Error parsing JSON: ' + str(e)))
                    return None
                if type(self.data) is not dict:
                    print(red('Error parsing JSON from file ' + path))
                    return None
                handle.close()
        except IOError as e:
            print(red('Could not load file: ' + str(e)))
            return None

        return True

    def save(self, d = None, f = None):
        if not d:
            d = self.data
        if not f:
            f = self.filename
        try:
            with open(self.project_dir + '/' + f, 'w') as handle:
                json.dump(d, handle, indent=4)
        except IOError as e:
            print(red('Could not save file: ' + str(e)))
            exit()
        else:
            print(green('Saved data to file: ' + f))

    ### Prompt/Deprompt functions ###
    def prompt_bool(self, p):
        if p is True:
            return 'y'
        if p is False:
            return 'n'
        return ''

    def deprompt_bool(self, p):
        if p.lower() in ['y','yes','true']:
            return True
        if p.lower() in ['n','no','false']:
            return False
        return None

lz = Lazyconf(os.path.dirname(__file__))
lz.config()
<commit_msg>Lists and bools now validate with regex<commit_after>import os, json, re
from fabric.api import *
from distutils.util import strtobool
from fabric.colors import green, red

class Lazyconf():

    def load_only_config(self):

        # If we're loading the config to actually serve, we can't fall back.
        self.load_data(False)

    def config(self):

        # If we're configuring, load schema first for internals
        self.load_data(True)
        self.__cfg(self.data, 'config')
        self.save()

    def save_schema(self):
        d = self.data
        self.__sv_schema(d)

    def __sv_schema(self, d):
        for k,v in d.iteritems():
            if t is dict:
                self.__sv_schema(d)
            elif t is bool:
                v = False
            elif t is int:
                v = 0
            else:
                v = ''
        return

    def __get_list(self, k):
        if k in self.lists.keys():
            return self.lists[k]
        return None

    def __get_list_value(self, k, l):
        if k in l.keys():
            return l[k]
        return ""

    def __get_list_key(self, v, l):
        for k,val in l.iteritems():
            if v == val:
                return k
        return ""

    def __get_label(self, l):
        if l in self.labels.keys():
            return self.labels[l]
        return l

    def __cfg(self, d, key_string):

        if '_enabled' in d.keys():
            s = '.'.join([key_string,'_enabled'])
            release = prompt(self.__get_label(s) + ' (y/n)?', default = self.prompt_bool(d['_enabled']), validate=r'^(y|n)$')
            d['_enabled'] = self.deprompt_bool(release)
            if d['_enabled'] is False:
                return

        for k, v in d.iteritems():
            if key_string == 'config._internal':
                return

            if k == '_enabled':
                continue

            t = type(v)
            s = '.'.join([key_string,k])

            list = self.__get_list(s)
            if list:
                choices = '(' + ','.join(list.keys()) + ')'
                re_choices = '^(' + '|'.join(list.keys()) + ')$'.encode('string_escape');

                d[k] = self.__get_list_value(prompt(self.__get_label(s) + ' ' + choices + ':', default = self.__get_list_key(v,list), validate=re_choices), list)
            elif t is dict:
                self.__cfg(v, s)
            elif t is str or t is unicode:
                d[k] = prompt(self.__get_label(s) + ':', default = v)
            elif t is bool:
                release = prompt(self.__get_label(s) + ' (y/n)?', default = self.prompt_bool(v), validate=r'^(y|n)$')
                d[k] = self.deprompt_bool(release)
            elif t is int:
                d[k] = prompt(self.__get_label(s) + ':', default = v)

    def __init__(self, project_dir, filename = 'lazyconf.json'):

        ### Project directory and filename ###
        if len(project_dir) == 0:
            project_dir = '.'

        self.project_dir = project_dir

        self.filename = filename

        self.data = {}

        self.labels = {}

        self.lists = {}

    def load_schema(self):

        # Set the path to the values from __init__.
        path = self.project_dir + '/' + self.filename

        # If we can't find a schema file in that folder, load the default schema.
        if not self.load_file(path, True):
            print(red("Could not load schema from " + path + ". Falling back to default..."))

            bkp_path = './' + self.filename

            # If we can't find the default schema, we cannot continue.
            if not self.load_file(bkp_path, True):
                print(red("Fatal: Could not load default schema from " + bkp_path + ". Aborting..."))
                exit()
            else:
                print(green("Loaded default schema."))
        else:
            print(green("Loaded schema from " + path + ".schema"))

    def load_data(self, fallback = True):

        # Set the path to the values from __init__.
        path = self.project_dir + '/' + self.filename

        # If we can't find the file in the folder and we're falling back ot schema, call load_schema.
        if not self.load_file(path):
            if fallback:
                self.load_schema()
            else:
                print(red("Fatal: Could not load schema from " + path + ". Aborting..."))
                exit()

    def load_file(self, path, schema = False):
        if schema:
            path += '.schema'
        try:
            with open(path) as handle:

                try:
                    self.data = json.load(handle)
                    self.labels = self.data['_internal']['labels']
                    self.lists = self.data['_internal']['lists']
                except ValueError as e:
                    print(red('Error parsing JSON: ' + str(e)))
                    return None
                if type(self.data) is not dict:
                    print(red('Error parsing JSON from file ' + path))
                    return None
                handle.close()
        except IOError as e:
            print(red('Could not load file: ' + str(e)))
            return None

        return True

    def save(self, d = None, f = None):
        if not d:
            d = self.data
        if not f:
            f = self.filename
        try:
            with open(self.project_dir + '/' + f, 'w') as handle:
                json.dump(d, handle, indent=4)
        except IOError as e:
            print(red('Could not save file: ' + str(e)))
            exit()
        else:
            print(green('Saved data to file: ' + f))

    ### Prompt/Deprompt functions ###
    def prompt_bool(self, p):
        if p is True:
            return 'y'
        if p is False:
            return 'n'
        return ''

    def deprompt_bool(self, p):
        if p == 'y':
            return True
        if p == 'n':
            return False
        return None

lz = Lazyconf(os.path.dirname(__file__))
lz.config()
<|endoftext|>


<commit_before>"""Leetcode 139. Word Break
Medium

URL: https://leetcode.com/problems/word-break/

Given a non-empty string s and a dictionary wordDict containing a list of
non-empty words, determine if s can be segmented into a space-separated sequence
of one or more dictionary words.

Note:
- The same word in the dictionary may be reused multiple times in the segmentation.
- You may assume the dictionary does not contain duplicate words.

Example 1:
Input: s = "leetcode", wordDict = ["leet", "code"]
Output: true
Explanation: Return true because "leetcode" can be segmented as "leet code".

Example 2:
Input: s = "applepenapple", wordDict = ["apple", "pen"]
Output: true
Explanation: Return true because "applepenapple" can be segmented as "apple pen apple".
             Note that you are allowed to reuse a dictionary word.

Example 3:
Input: s = "catsandog", wordDict = ["cats", "dog", "sand", "and", "cat"]
Output: false
"""

class Solution(object):
    def wordBreak(self, s, wordDict):
        """
        :type s: str
        :type wordDict: List[str]
        :rtype: bool
        """
        pass

def main():
    pass

if __name__ == '__main__':
    main()
<commit_msg>Complete DP sol<commit_after>"""Leetcode 139. Word Break
Medium

URL: https://leetcode.com/problems/word-break/

Given a non-empty string s and a dictionary wordDict containing a list of
non-empty words, determine if s can be segmented into a space-separated sequence
of one or more dictionary words.

Note:
- The same word in the dictionary may be reused multiple times in the segmentation.
- You may assume the dictionary does not contain duplicate words.

Example 1:
Input: s = "leetcode", wordDict = ["leet", "code"]
Output: true
Explanation: Return true because "leetcode" can be segmented as "leet code".

Example 2:
Input: s = "applepenapple", wordDict = ["apple", "pen"]
Output: true
Explanation: Return true because "applepenapple" can be segmented as "apple pen apple".
             Note that you are allowed to reuse a dictionary word.

Example 3:
Input: s = "catsandog", wordDict = ["cats", "dog", "sand", "and", "cat"]
Output: false
"""

class SolutionDp(object):
    def wordBreak(self, s, wordDict):
        """
        :type s: str
        :type wordDict: List[str]
        :rtype: bool
        """
        # Apply DP with tabular T.
        n = len(s)
        T = [False] * (n + 1)

        # Empty string is breakable.
        T[0] = True

        for i in range(1, n + 1):
            for w in wordDict:
                # Check index is valid, the left & right parts are matched.
                if i - len(w) >= 0 and T[i - len(w)] and s[(i - len(w)):i] == w:
                    T[i] = True
                    break

        return T[-1]

def main():
    # Ans: True
    s = "leetcode"
    wordDict = ["leet", "code"]
    print SolutionDp().wordBreak(s, wordDict)

    # Ans: True
    s = "applepenapple"
    wordDict = ["apple", "pen"]
    print SolutionDp().wordBreak(s, wordDict)

    # Ans: False
    s = "catsandog"
    wordDict = ["cats", "dog", "sand", "and", "cat"]
    print SolutionDp().wordBreak(s, wordDict)

    # Ans: True
    s = "aaaaaaa"
    wordDict = ["aaaa","aaa"]
    print SolutionDp().wordBreak(s, wordDict)

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>'''Learning_automata.py is a module containing some learning automata.'''

# Written by: Steven Porretta.

# For now it only contains a Tsetlin 2N,2 automaton.

from random import uniform
import numpy as np

# Write a Tsetlin automaton.
# A Tsetlin automaton is defined by N memory states, R actions,
# and c penalties.
#
# N is expected to be a whole number indicating memory depth.
# R is expected to be a while number indicating the number of,
# actions.
# c is a vector quantity indicating the penalties for actions
# 1 to i.

# There should be an abstract class that defines the important parameters
# of a learning automaton, shared by all learning automata.

class LA(object):
    '''A learning automaton.'''

    def __init__(self):
        '''Set up states, penalties, etc.'''
        pass

    def next_state_on_reward(self):
        '''Define the state translation, or action selection,
           on reward.'''
        pass

    def next_state_on_penalty(self):
        '''Define the state translation, or action selection,
           on penalty.'''
        pass

    def environment_response(self):
        '''Determine the reward, and then call for the appropriate
           response.'''
        pass

    def simulate(self):
        '''Define the simulation for the automaton.'''
        pass

class Tsetlin(LA):
    '''Tsetlin automata. N, R positive integers. c a structure
       of penalties from action 1 to action i. First state starts at 1 not
       0.'''

    def __init__(self, N, R, c):
        '''initialize a Tsetlin automaton with N memory states,
           R actions, and c penalty probabilities for each R.'''
        self.N = 2 * N
        self.n = N
        self.R = R
        self.c = c
        # define the initial state as the state leaning toward action one.
        self.current_state = int(self.N / R)
        self.states = []
        self.actions = np.zeros(self.R)
        self.action_average = 0

    # Find the next state given that the teacher rewarded.
    def next_state_on_reward(self):
        '''Find the next state of the learner, given that the teacher
           rewarded.'''

        # I need to know what state is the next loopback state.
        # There are R actions, and therefore there are 2N/R memory blocks.
        # If the current state is <= 2N/R in action 1
        # If 2N/R < current state < 2(2N/R) in action 2

        # If the mod is 1, then the current state is the loopback state.
        if (self.current_state % (self.N / self.R) != 1):
            self.current_state -= 1

    # Find the next state given that the teacher penalized.
    def next_state_on_penalty(self):
        '''Find the next state of the learner, given that the teacher
           penalized.'''

        # The hard part here is not looping back, but instead jumping to
        # the next 2N / R state.

        if(self.current_state % (self.N / self.R) != 0):
            self.current_state += 1
        elif(self.current_state % (self.N / self.R) == 0):
            # Don't really add states, just cycle through N, 2N, 4N, etc.
            if(self.current_state != self.N):
                a = (self.N / self.R) % self.N
                self.current_state = a + self.current_state
            else:
                self.current_state = self.N / self.R

    # Determine the next state as the teacher.
    def environment_response(self):
        '''Determine the next state of the learner from the perspective
        of the teacher.'''
        response = uniform(0, 1)
        # The penality index is the index of the penalty array.
        # Do 2 action for now
        penalty_index = 1
        if(self.current_state <= self.n):
            self.actions[0] += 1
            penalty_index = 0
        else:
            self.actions[1] += 1

        if(response > self.c[penalty_index]):
            # Reward.
            self.next_state_on_reward()
        else:
            # Penalty.
            self.next_state_on_penalty()

    # for action counting pretend it is always 2 action for now.
    def action_count(self):
        if(self.current_state <= self.n):
            # In action 1.
            self.actions[0] += 1
        else:
            self.actions[1] += 1

    # Run a simulation of a Tsetlin automaton.
    # It might be best to put this function somewhere else...
    def simulate(self, n, ensemble_size):
        '''Run a simulation of a Tsetlin automaton for a size of
           experiment, n, and a number of experiments, ensemble_size.'''

        for i in range(ensemble_size):
            self.states.append(np.zeros(self.N))
            for j in range(n):
                self.environment_response()

        self.action_average = self.actions / (n * ensemble_size)

class Krylov(Tsetlin):
    '''Krylov automaton.  Inherits from the Tsetlin automaton.'''

    def __init__(self, N, R, c):
        '''The constructor of a Krylov is exactly the same as the Tsetlin.
           The only difference between the machines is how penalties are
           handled.'''

        Tsetlin.__init__(self, N, R, c)

    def next_state_on_penalty(self):
        '''Find the next state of the learner, given that the teacher
           penalized.'''

        # If this number is greater than 0.5, then penalize the learner.
        is_penalty = uniform(0, 1)

        if(is_penalty >= 0.5):
            Tsetlin.next_state_on_penalty(self)
        else:
            Tsetlin.next_state_on_reward(self)

class Linear(object):
    '''The Linear Reward-Inaction model (for now).'''

    # This time separate all concerns, no vectors.
    def __init__(self, c1, c2):
        self.c1 = c1
        self.p1 = 0.5  # Always start in the 0.5 state at the beginning.
        self.c2 = c2
        self.p2 = 0.5  # Always start in the 0.5 state at the beginning.
        self.k_r = 0  # k_r = 1 - lambdaR

    def next_action(self):
        randy = uniform(0, 1)  # Throwback to Archer.
        if(randy < self.p1):
            # Choose action 1.
            return 1
        return 2

    def environment_response(self):
        # For now let's always give a reward.
        return 0

    def do_reward(self, action):
        if(action == 2):
            self.p1 = self.k * self.p1
        else:
            self.p1 = (1 - self.k) * self.p2
        self.p2 = 1 - self.p1

    def do_penalty(self):
        pass

    def simulate(self):
        a1_counter = 0
        a2_counter = 0

        while(True):
            if(self.p1 or self.p2 > 0.4):
                # Close enough to 1.
                # Treat as convergence.
                break
            if(self.next_action() == 1):
                # Do action 1
            else:
                # Do action 2
        return 0  # Return the action average.
<commit_msg>commit before leaving.<commit_after>'''Learning_automata.py is a module containing some learning automata.'''

# Written by: Steven Porretta.

# For now it only contains a Tsetlin 2N,2 automaton.

from random import uniform
import numpy as np

# Write a Tsetlin automaton.
# A Tsetlin automaton is defined by N memory states, R actions,
# and c penalties.
#
# N is expected to be a whole number indicating memory depth.
# R is expected to be a while number indicating the number of,
# actions.
# c is a vector quantity indicating the penalties for actions
# 1 to i.

# There should be an abstract class that defines the important parameters
# of a learning automaton, shared by all learning automata.

class LA(object):
    '''A learning automaton.'''

    def __init__(self):
        '''Set up states, penalties, etc.'''
        pass

    def next_state_on_reward(self):
        '''Define the state translation, or action selection,
           on reward.'''
        pass

    def next_state_on_penalty(self):
        '''Define the state translation, or action selection,
           on penalty.'''
        pass

    def environment_response(self):
        '''Determine the reward, and then call for the appropriate
           response.'''
        pass

    def simulate(self):
        '''Define the simulation for the automaton.'''
        pass

class Tsetlin(LA):
    '''Tsetlin automata. N, R positive integers. c a structure
       of penalties from action 1 to action i. First state starts at 1 not
       0.'''

    def __init__(self, N, R, c):
        '''initialize a Tsetlin automaton with N memory states,
           R actions, and c penalty probabilities for each R.'''
        self.N = 2 * N
        self.n = N
        self.R = R
        self.c = c
        # define the initial state as the state leaning toward action one.
        self.current_state = int(self.N / R)
        self.states = []
        self.actions = np.zeros(self.R)
        self.action_average = 0

    # Find the next state given that the teacher rewarded.
    def next_state_on_reward(self):
        '''Find the next state of the learner, given that the teacher
           rewarded.'''

        # I need to know what state is the next loopback state.
        # There are R actions, and therefore there are 2N/R memory blocks.
        # If the current state is <= 2N/R in action 1
        # If 2N/R < current state < 2(2N/R) in action 2

        # If the mod is 1, then the current state is the loopback state.
        if (self.current_state % (self.N / self.R) != 1):
            self.current_state -= 1

    # Find the next state given that the teacher penalized.
    def next_state_on_penalty(self):
        '''Find the next state of the learner, given that the teacher
           penalized.'''

        # The hard part here is not looping back, but instead jumping to
        # the next 2N / R state.

        if(self.current_state % (self.N / self.R) != 0):
            self.current_state += 1
        elif(self.current_state % (self.N / self.R) == 0):
            # Don't really add states, just cycle through N, 2N, 4N, etc.
            if(self.current_state != self.N):
                a = (self.N / self.R) % self.N
                self.current_state = a + self.current_state
            else:
                self.current_state = self.N / self.R

    # Determine the next state as the teacher.
    def environment_response(self):
        '''Determine the next state of the learner from the perspective
        of the teacher.'''
        response = uniform(0, 1)
        # The penality index is the index of the penalty array.
        # Do 2 action for now
        penalty_index = 1
        if(self.current_state <= self.n):
            self.actions[0] += 1
            penalty_index = 0
        else:
            self.actions[1] += 1

        if(response > self.c[penalty_index]):
            # Reward.
            self.next_state_on_reward()
        else:
            # Penalty.
            self.next_state_on_penalty()

    # for action counting pretend it is always 2 action for now.
    def action_count(self):
        if(self.current_state <= self.n):
            # In action 1.
            self.actions[0] += 1
        else:
            self.actions[1] += 1

    # Run a simulation of a Tsetlin automaton.
    # It might be best to put this function somewhere else...
    def simulate(self, n, ensemble_size):
        '''Run a simulation of a Tsetlin automaton for a size of
           experiment, n, and a number of experiments, ensemble_size.'''

        for i in range(ensemble_size):
            self.states.append(np.zeros(self.N))
            for j in range(n):
                self.environment_response()

        self.action_average = self.actions / (n * ensemble_size)

class Krylov(Tsetlin):
    '''Krylov automaton.  Inherits from the Tsetlin automaton.'''

    def __init__(self, N, R, c):
        '''The constructor of a Krylov is exactly the same as the Tsetlin.
           The only difference between the machines is how penalties are
           handled.'''

        Tsetlin.__init__(self, N, R, c)

    def next_state_on_penalty(self):
        '''Find the next state of the learner, given that the teacher
           penalized.'''

        # If this number is greater than 0.5, then penalize the learner.
        is_penalty = uniform(0, 1)

        if(is_penalty >= 0.5):
            Tsetlin.next_state_on_penalty(self)
        else:
            Tsetlin.next_state_on_reward(self)

class Linear(object):
    '''The Linear Reward-Inaction model (for now).'''

    # This time separate all concerns, no vectors.
    def __init__(self, c1, c2):
        self.c1 = c1
        self.p1 = 0.5  # Always start in the 0.5 state at the beginning.
        self.c2 = c2
        self.p2 = 0.5  # Always start in the 0.5 state at the beginning.
        self.k_r = 0  # k_r = 1 - lambdaR

    def next_action(self):
        randy = uniform(0, 1)  # Throwback to Archer.
        if(randy < self.p1):
            # Choose action 1.
            return 1
        return 2

    def environment_response(self):
        # For now let's always give a reward.
        return 0

    def do_reward(self, action):
        if(action == 2):
            self.p1 = self.k * self.p1
        else:
            self.p1 = (1 - self.k) * self.p2
        self.p2 = 1 - self.p1

    def do_penalty(self):
        pass

    def simulate(self):
        a1_counter = 0
        a2_counter = 0
        action = 0

        while(True):
            if(self.p1 or self.p2 > 0.4):
                # Close enough to 1.
                # Treat as convergence.
                break
            action = self.next_action()
            if(action == 1):
                # Do action 1
                a1_counter += 1
            else:
                # Do action 2
                a2_counter += 1
        return 0  # Return the action average.
<|endoftext|>


<commit_before>#!/usr/bin/python
import ctypes
import re

""" since Python has no unsigned int builtin: """
class ureg(object):
	a, b = 0, 0
	def __setattr__(self, attr, val):
		self.__dict__[attr] = ctypes.c_uint32(val).value

regs = ureg()
prog = []
cir, eip = 0, 0

with open('in.txt', 'r') as f:
	for line in f:
		line = line.strip()

		if __debug__: print line
		prog.append(line)

def dumpregs():
	print("Registers: a: {0: <10d} b: {1:<10d}".format(regs.a, regs.b))

print("Running...")
done = False
while(not done):
	cir = eip
	eip += 1

	if(cir >= len(prog)):
		print("Done. Out of instructions")
		dumpregs()
		done = True
		break

	act = prog[cir]
	if __debug__:
		print("[{0:4d}]: {1:30s}".format(cir, act)),
		dumpregs()

	if(act.split()[0] == 'inc'):
		setattr(regs, act.split()[1], getattr(regs, act.split()[1]) + 1)
		continue
	elif(act.split()[0] == 'tpl'):
		setattr(regs, act.split()[1], getattr(regs, act.split()[1]) * 3)
		continue
	elif(act.split()[0] == 'hlf'):
		setattr(regs, act.split()[1], int(getattr(regs, act.split()[1]) / 2) )
		continue
	elif(act.split()[0] == 'jmp'):
		eip = cir + int(act.split()[1])
		continue
	elif(act.split()[0] == 'jio'):
		parts = re.split(' |,', act)
		comp = parts[1]
		trg = int(parts[3])
		if(getattr(regs, comp) == 1):
			eip += trg - 1
		continue
	elif(act.split()[0] == 'jie'):
		parts = re.split(' |,', act)
		comp = parts[1]
		trg = int(parts[3])
		if(getattr(regs, comp) % 2 == 0):
			eip += trg - 1
		continue

r"""
--- Day 23: Opening the Turing Lock ---

Little Jane Marie just got her very first computer for Christmas from some unknown benefactor. It comes with instructions and an example program, but the computer itself seems to be malfunctioning. She's curious what the program does, and would like you to help her run it.

The manual explains that the computer supports two registers and six instructions (truly, it goes on to remind the reader, a state-of-the-art technology). The registers are named a and b, can hold any non-negative integer, and begin with a value of 0. The instructions are as follows:

    hlf r sets register r to half its current value, then continues with the next instruction.
    tpl r sets register r to triple its current value, then continues with the next instruction.
    inc r increments register r, adding 1 to it, then continues with the next instruction.
    jmp offset is a jump; it continues with the instruction offset away relative to itself.
    jie r, offset is like jmp, but only jumps if register r is even ("jump if even").
    jio r, offset is like jmp, but only jumps if register r is 1 ("jump if one", not odd).

All three jump instructions work with an offset relative to that instruction. The offset is always written with a prefix + or - to indicate the direction of the jump (forward or backward, respectively). For example, jmp +1 would simply continue with the next instruction, while jmp +0 would continuously jump back to itself forever.

The program exits when it tries to run an instruction beyond the ones defined.

For example, this program sets a to 2, because the jio instruction causes it to skip the tpl instruction:

inc a
jio a, +2
tpl a
inc a

What is the value in register b when the program in your puzzle input is finished executing?
"""
<commit_msg>Level 23, 2/2 solved<commit_after>#!/usr/bin/python
""" run as `python -O level_23.py` to disable debug garbage """
import ctypes
import re

""" since Python has no unsigned int builtin: """
class ureg(object):
	a, b = 0, 0
	def __setattr__(self, attr, val):
		self.__dict__[attr] = ctypes.c_uint32(val).value
""" turned out to be a surplus code :) """

regs = ureg()
prog = []

with open('in.txt', 'r') as f:
	for line in f:
		line = line.strip()

		if __debug__: print line
		prog.append(line)

def dumpregs():
	print("Registers: a: {0: <10d} b: {1:<10d}".format(regs.a, regs.b))

def runprog():
	cir, eip = 0, 0
	print("Running...")
	done = False
	while(not done):
		cir = eip
		eip += 1

		if(cir >= len(prog)):
			print("Done. Out of instructions")
			dumpregs()
			done = True
			break

		act = prog[cir]
		if __debug__:
			print("[{0:4d}]: {1:30s}".format(cir, act)),
			dumpregs()

		if(act.split()[0] == 'inc'):
			setattr(regs, act.split()[1], getattr(regs, act.split()[1]) + 1)
			continue
		elif(act.split()[0] == 'tpl'):
			setattr(regs, act.split()[1], getattr(regs, act.split()[1]) * 3)
			continue
		elif(act.split()[0] == 'hlf'):
			setattr(regs, act.split()[1], int(getattr(regs, act.split()[1]) / 2) )
			continue
		elif(act.split()[0] == 'jmp'):
			eip = cir + int(act.split()[1])
			continue
		elif(act.split()[0] == 'jio'):
			parts = re.split(' |,', act)
			comp = parts[1]
			trg = int(parts[3])
			if(getattr(regs, comp) == 1):
				eip += trg - 1
			continue
		elif(act.split()[0] == 'jie'):
			parts = re.split(' |,', act)
			comp = parts[1]
			trg = int(parts[3])
			if(getattr(regs, comp) % 2 == 0):
				eip += trg - 1
			continue

runprog()

regs.a = 1
regs.b = 0

runprog()

r"""
--- Day 23: Opening the Turing Lock ---

Little Jane Marie just got her very first computer for Christmas from some unknown benefactor. It comes with instructions and an example program, but the computer itself seems to be malfunctioning. She's curious what the program does, and would like you to help her run it.

The manual explains that the computer supports two registers and six instructions (truly, it goes on to remind the reader, a state-of-the-art technology). The registers are named a and b, can hold any non-negative integer, and begin with a value of 0. The instructions are as follows:

    hlf r sets register r to half its current value, then continues with the next instruction.
    tpl r sets register r to triple its current value, then continues with the next instruction.
    inc r increments register r, adding 1 to it, then continues with the next instruction.
    jmp offset is a jump; it continues with the instruction offset away relative to itself.
    jie r, offset is like jmp, but only jumps if register r is even ("jump if even").
    jio r, offset is like jmp, but only jumps if register r is 1 ("jump if one", not odd).

All three jump instructions work with an offset relative to that instruction. The offset is always written with a prefix + or - to indicate the direction of the jump (forward or backward, respectively). For example, jmp +1 would simply continue with the next instruction, while jmp +0 would continuously jump back to itself forever.

The program exits when it tries to run an instruction beyond the ones defined.

For example, this program sets a to 2, because the jio instruction causes it to skip the tpl instruction:

inc a
jio a, +2
tpl a
inc a

What is the value in register b when the program in your puzzle input is finished executing?

--- Part Two ---

The unknown benefactor is very thankful for releasi-- er, helping little Jane Marie with her computer. Definitely not to distract you, what is the value in register b after the program is finished executing if register a starts as 1 instead?
"""
<|endoftext|>


<commit_before>from django.db import models
from django.core.urlresolvers import reverse
from localflavor.generic.models import IBANField

class Account(models.Model):
    first_name = models.CharField(max_length=200)
    last_name = models.CharField(max_length=200)
    iban_number = IBANField()

    def __unicode__(self):
        return self.name

    def get_absolute_url(self):
        return reverse('main:account_edit', kwargs={'pk': self.pk})
<commit_msg>Added __str__ method and verbose_name_plural to main:Account model<commit_after>from django.db import models
from django.core.urlresolvers import reverse
from localflavor.generic.models import IBANField

class Account(models.Model):
    first_name = models.CharField(max_length=200)
    last_name = models.CharField(max_length=200)
    iban_number = IBANField()

    class Meta:
        verbose_name_plural = "accounts"

    def __unicode__(self):
        return self.first_name

    def __str__(self):
        return self.first_name

    def get_absolute_url(self):
        return reverse('main:account_edit', kwargs={'pk': self.pk})
<|endoftext|>


<commit_before>import numpy as np

class Maze():

    dir1 = np.array([+1,  0,  0, -1], dtype=int)
    dir2 = np.array([ 0, +1, -1,  0], dtype=int)

    def __init__(self, start, target, pMap):

        self.start  = start
        self.target = target

        self.pMap = np.zeros((pMap.shape[0]+1, pMap.shape[1]+1), dtype=bool)
        self.pMap[:~0, :~0] = pMap[:, :]  # Make map with 0-padding.
        self.nMapWidth = pMap.shape[1]

        self.directions = np.zeros((4, 2), dtype=int)

    def solve(self, nOutBufferSize):

        self.currentBest = np.inf
        self.currentBestSnake = []
        self.nOutBufferSize = nOutBufferSize
        self.set_direction(self.start)
        self.forward([self.start])

        if np.isinf(self.currentBest):
            return -1
        else:
            return self.currentBest - 1, self.currentBestSnake[1:]

    def forward(self, snake):

        # print np.array(snake).tolist()  # Print current snake.
        for prevPos in snake[:~0]:
            if (snake[~0] == prevPos).all():
                # Been here before.
                # print "Been here before: %s" % str(prevPos)
                return
        best_possibility = len(snake) + np.abs(self.target - snake[~0]).sum()
        if best_possibility >= self.currentBest:
            # A better solution cannot be found anymore.
            # print "A better solution cannot be found anymore."
            return
        elif best_possibility > self.nOutBufferSize:
            # Snake is too long.
            # print "Snake is too long."
            return
        elif (snake[~0] == self.target).all():
            # Victory!
            # print "Victory!"
            self.currentBest = len(snake)
            self.currentBestSnake = list(snake)
            return

        for direction in self.directions:
            newPos = snake[~0] + direction
            if self.pMap[newPos[0], newPos[1]]:  # if open path
                self.forward(snake + [newPos])  # continue moving

    def set_direction(self, head):

        direction = self.target - head
        i = np.argmax(np.abs(direction))

        if direction[i] >= 0:
            self.directions[:, i] = +self.dir1
        else:
            self.directions[:, i] = -self.dir1
        if direction[not i] >= 0:
            self.directions[:, not i] = +self.dir2
        else:
            self.directions[:, not i] =  -self.dir2
<commit_msg>only return the path or None if no path was found<commit_after>import numpy as np

class Maze():

    dir1 = np.array([+1,  0,  0, -1], dtype=int)
    dir2 = np.array([ 0, +1, -1,  0], dtype=int)

    def __init__(self, start, target, pMap):

        self.start  = start
        self.target = target

        self.pMap = np.zeros((pMap.shape[0]+1, pMap.shape[1]+1), dtype=bool)
        self.pMap[:~0, :~0] = pMap[:, :]  # Make map with 0-padding.
        self.nMapWidth = pMap.shape[1]

        self.directions = np.zeros((4, 2), dtype=int)

    def solve(self, nOutBufferSize):

        self.currentBest = np.inf
        self.currentBestSnake = []
        self.nOutBufferSize = nOutBufferSize
        self.set_direction(self.start)
        self.forward([self.start])

        if np.isinf(self.currentBest):
            return None
        else:
            return self.currentBestSnake[1:]

    def forward(self, snake):

        # print np.array(snake).tolist()  # Print current snake.
        for prevPos in snake[:~0]:
            if (snake[~0] == prevPos).all():
                # Been here before.
                # print "Been here before: %s" % str(prevPos)
                return
        best_possibility = len(snake) + np.abs(self.target - snake[~0]).sum()
        if best_possibility >= self.currentBest:
            # A better solution cannot be found anymore.
            # print "A better solution cannot be found anymore."
            return
        elif best_possibility > self.nOutBufferSize:
            # Snake is too long.
            # print "Snake is too long."
            return
        elif (snake[~0] == self.target).all():
            # Victory!
            # print "Victory!"
            self.currentBest = len(snake)
            self.currentBestSnake = list(snake)
            return

        for direction in self.directions:
            newPos = snake[~0] + direction
            if self.pMap[newPos[0], newPos[1]]:  # if open path
                self.forward(snake + [newPos])  # continue moving

    def set_direction(self, head):

        direction = self.target - head
        i = np.argmax(np.abs(direction))

        if direction[i] >= 0:
            self.directions[:, i] = +self.dir1
        else:
            self.directions[:, i] = -self.dir1
        if direction[not i] >= 0:
            self.directions[:, not i] = +self.dir2
        else:
            self.directions[:, not i] =  -self.dir2
<|endoftext|>


<commit_before>import logging

from math import ceil
from time import mktime

from Common import DB
from Sharding import Sharding

class Replset:
    def __init__(self, db, user, password, authdb, max_lag_secs):
        self.db           = db
        self.user         = user
        self.password     = password
        self.authdb       = authdb
        self.max_lag_secs = max_lag_secs

        # Get a DB connection
        try:
            if self.db.__class__.__name__ == "DB":
                self.connection = self.db.connection()
            else:
                raise Exception, "'db' field is an instance of %s, not 'DB'!" % self.db.__class__.__name__, None
        except Exception, e:
            logging.fatal("Could not get DB connection! Error: %s" % e)
            raise e

    def close(self):
        pass

    def get_rs_status(self):
        try:
            return self.db.admin_command('replSetGetStatus')
        except Exception, e:
            raise Exception, "Error getting replica set status! Error: %s" % e, None

    def get_rs_config(self):
        try:
            if self.db.server_version() >= tuple("3.0.0".split(".")):
                output = self.db.admin_command('replSetGetConfig')
                return output['config']
            else:
                return self.connection['local'].system.replset.find_one()
        except Exception, e:
            raise Exception, "Error getting replica set config! Error: %s" % e, None

    def find_secondary(self):
        rs_status    = self.get_rs_status()
        rs_config    = self.get_rs_config()
        rs_name      = rs_status['set']
        quorum_count = ceil(len(rs_status['members']) / 2.0)

        primary = None
        for member in rs_status['members']:
            if member['stateStr'] == 'PRIMARY' and member['health'] > 0:
                primary = {
                    'host': member['name'],
                    'optime': member['optimeDate']
                }
                optime = member['optime']
                if 'ts' in member['optime']:
                    optime = member['optime']['ts']
                logging.info("Found PRIMARY: %s/%s with optime %s" % (
                    rs_name,
                    member['name'],
                    str(optime)
                ))
        if primary is None:
            logging.fatal("Unable to locate a PRIMARY member for replset %s, giving up" % rs_name)
            raise Exception, "Unable to locate a PRIMARY member for replset %s, giving up" % rs_name, None

        secondary = None
        for member in rs_status['members']:
            if member['stateStr'] == 'SECONDARY' and member['health'] > 0:
                score       = self.max_lag_secs * 10
                score_scale = 100 / score
                log_data    = {}

                hidden_weight = 0.20
                for member_config in rs_config['members']:
                    if member_config['host'] == member['name']:
                        if 'hidden' in member_config and member_config['hidden'] == True:
                            score += (score * hidden_weight)
                            log_data['hidden'] = True
                        if 'priority' in member_config:
                            log_data['priority'] = int(member_config['priority'])
                            if member_config['priority'] > 0:
                                score = score - member_config['priority']
                        break

                rep_lag = (mktime(primary['optime'].timetuple()) - mktime(member['optimeDate'].timetuple()))
                score = ceil((score - rep_lag) * score_scale)
                if rep_lag < self.max_lag_secs:
                    if secondary is None or score > secondary['score']:
                        secondary = {
                            'replSet': rs_name,
                            'count': 1 if secondary is None else secondary['count'] + 1,
                            'host': member['name'],
                            'optime': member['optimeDate'],
                            'score': score
                        }
                    log_msg = "Found SECONDARY %s/%s" % (rs_name, member['name'])
                else:
                    log_msg = "Found SECONDARY %s/%s with too-high replication lag! Skipping" % (rs_name, member['name'])

                log_data['optime'] = member['optime']
                if 'ts' in member['optime']:
                    log_data['optime'] = member['optime']['ts']
                log_data['score']  = int(score)
                logging.info("%s: %s" % (log_msg, str(log_data)))
        if secondary is None or (secondary['count'] + 1) < quorum_count:
            logging.fatal("Not enough secondaries in replset %s to take backup! Num replset members: %i, required quorum: %i" % (
                rs_name,
                secondary['count'] + 1,
                quorum_count
            ))
            raise Exception, "Not enough secondaries in replset %s to safely take backup!" % rs_name, None

        logging.info("Choosing SECONDARY %s for replica set %s (score: %i)" % (secondary['host'], rs_name, secondary['score']))
        return secondary

class ReplsetSharded:
    def __init__(self, sharding, db, user, password, authdb, max_lag_secs):
        self.sharding     = sharding
        self.db           = db
        self.user         = user
        self.password     = password
        self.authdb       = authdb
        self.max_lag_secs = max_lag_secs

        self.replset = None

        # Check Sharding class:
        if not self.sharding.__class__.__name__ == "Sharding":
            raise Exception, "'sharding' field is an instance of %s, not 'Sharding'!" % self.sharding.__class__.__name__, None

        # Get a DB connection
        try:
            if self.db.__class__.__name__ == "DB":
                self.connection = self.db.connection()
                if not self.connection.is_mongos:
                    raise Exception, 'MongoDB connection is not to a mongos!', None
            else:
                raise Exception, "'db' field is an instance of %s, not 'DB'!" % self.db.__class__.__name__, None
        except Exception, e:
            logging.fatal("Could not get DB connection! Error: %s" % e)
            raise e

    def find_secondaries(self):
        shard_secondaries = {}
        if self.sharding:
            for shard in self.sharding.shards():
                shard_name, members = shard['host'].split('/')
                host, port          = members.split(',')[0].split(":")

                replset_db   = DB(host, port, self.user, self.password, self.authdb)
                self.replset = Replset(replset_db, self.user, self.password, self.authdb, self.max_lag_secs)
                secondary    = self.replset.find_secondary()
                shard_secondaries[shard_name] = secondary

                self.replset.close()
                replset_db.close()
        return shard_secondaries

    def close(self):
        if self.replset:
            self.replset.close()
<commit_msg>merging 2.4_optime_fix fix<commit_after>import logging

from math import ceil
from time import mktime

from Common import DB
from Sharding import Sharding

class Replset:
    def __init__(self, db, user, password, authdb, max_lag_secs):
        self.db           = db
        self.user         = user
        self.password     = password
        self.authdb       = authdb
        self.max_lag_secs = max_lag_secs

        # Get a DB connection
        try:
            if self.db.__class__.__name__ == "DB":
                self.connection = self.db.connection()
            else:
                raise Exception, "'db' field is an instance of %s, not 'DB'!" % self.db.__class__.__name__, None
        except Exception, e:
            logging.fatal("Could not get DB connection! Error: %s" % e)
            raise e

    def close(self):
        pass

    def get_rs_status(self):
        try:
            return self.db.admin_command('replSetGetStatus')
        except Exception, e:
            raise Exception, "Error getting replica set status! Error: %s" % e, None

    def get_rs_config(self):
        try:
            if self.db.server_version() >= tuple("3.0.0".split(".")):
                output = self.db.admin_command('replSetGetConfig')
                return output['config']
            else:
                return self.connection['local'].system.replset.find_one()
        except Exception, e:
            raise Exception, "Error getting replica set config! Error: %s" % e, None

    def find_secondary(self):
        rs_status    = self.get_rs_status()
        rs_config    = self.get_rs_config()
        rs_name      = rs_status['set']
        quorum_count = ceil(len(rs_status['members']) / 2.0)

        primary = None
        for member in rs_status['members']:
            if member['stateStr'] == 'PRIMARY' and member['health'] > 0:
                primary = {
                    'host': member['name'],
                    'optime': member['optimeDate']
                }
                optime = member['optime']
                if isinstance(member['optime'], dict) and 'ts' in member['optime']:
                    optime = member['optime']['ts']
                logging.info("Found PRIMARY: %s/%s with optime %s" % (
                    rs_name,
                    member['name'],
                    str(optime)
                ))
        if primary is None:
            logging.fatal("Unable to locate a PRIMARY member for replset %s, giving up" % rs_name)
            raise Exception, "Unable to locate a PRIMARY member for replset %s, giving up" % rs_name, None

        secondary = None
        for member in rs_status['members']:
            if member['stateStr'] == 'SECONDARY' and member['health'] > 0:
                score       = self.max_lag_secs * 10
                score_scale = 100 / score
                log_data    = {}

                hidden_weight = 0.20
                for member_config in rs_config['members']:
                    if member_config['host'] == member['name']:
                        if 'hidden' in member_config and member_config['hidden'] == True:
                            score += (score * hidden_weight)
                            log_data['hidden'] = True
                        if 'priority' in member_config:
                            log_data['priority'] = int(member_config['priority'])
                            if member_config['priority'] > 0:
                                score = score - member_config['priority']
                        break

                rep_lag = (mktime(primary['optime'].timetuple()) - mktime(member['optimeDate'].timetuple()))
                score = ceil((score - rep_lag) * score_scale)
                if rep_lag < self.max_lag_secs:
                    if secondary is None or score > secondary['score']:
                        secondary = {
                            'replSet': rs_name,
                            'count': 1 if secondary is None else secondary['count'] + 1,
                            'host': member['name'],
                            'optime': member['optimeDate'],
                            'score': score
                        }
                    log_msg = "Found SECONDARY %s/%s" % (rs_name, member['name'])
                else:
                    log_msg = "Found SECONDARY %s/%s with too-high replication lag! Skipping" % (rs_name, member['name'])

                log_data['optime'] = member['optime']
                if isinstance(member['optime'], dict) and 'ts' in member['optime']:
                    log_data['optime'] = member['optime']['ts']
                log_data['score']  = int(score)
                logging.info("%s: %s" % (log_msg, str(log_data)))
        if secondary is None or (secondary['count'] + 1) < quorum_count:
            logging.fatal("Not enough secondaries in replset %s to take backup! Num replset members: %i, required quorum: %i" % (
                rs_name,
                secondary['count'] + 1,
                quorum_count
            ))
            raise Exception, "Not enough secondaries in replset %s to safely take backup!" % rs_name, None

        logging.info("Choosing SECONDARY %s for replica set %s (score: %i)" % (secondary['host'], rs_name, secondary['score']))
        return secondary

class ReplsetSharded:
    def __init__(self, sharding, db, user, password, authdb, max_lag_secs):
        self.sharding     = sharding
        self.db           = db
        self.user         = user
        self.password     = password
        self.authdb       = authdb
        self.max_lag_secs = max_lag_secs

        self.replset = None

        # Check Sharding class:
        if not self.sharding.__class__.__name__ == "Sharding":
            raise Exception, "'sharding' field is an instance of %s, not 'Sharding'!" % self.sharding.__class__.__name__, None

        # Get a DB connection
        try:
            if self.db.__class__.__name__ == "DB":
                self.connection = self.db.connection()
                if not self.connection.is_mongos:
                    raise Exception, 'MongoDB connection is not to a mongos!', None
            else:
                raise Exception, "'db' field is an instance of %s, not 'DB'!" % self.db.__class__.__name__, None
        except Exception, e:
            logging.fatal("Could not get DB connection! Error: %s" % e)
            raise e

    def find_secondaries(self):
        shard_secondaries = {}
        if self.sharding:
            for shard in self.sharding.shards():
                shard_name, members = shard['host'].split('/')
                host, port          = members.split(',')[0].split(":")

                replset_db   = DB(host, port, self.user, self.password, self.authdb)
                self.replset = Replset(replset_db, self.user, self.password, self.authdb, self.max_lag_secs)
                secondary    = self.replset.find_secondary()
                shard_secondaries[shard_name] = secondary

                self.replset.close()
                replset_db.close()
        return shard_secondaries

    def close(self):
        if self.replset:
            self.replset.close()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""
.. module:: MyCapytain
   :platform: Unix, Windows
   :synopsis: Canonical Text Service and Canonical URN tool suite for Python

.. moduleauthor:: Thibault Clérice <leponteineptique@gmail.com>

"""

__name__ = "MyCapytain"
__version__ = "0.0.1"
__all__ = ["common", "endpoints", "resources"]<commit_msg>0.0.3<commit_after># -*- coding: utf-8 -*-
"""
.. module:: MyCapytain
   :platform: Unix, Windows
   :synopsis: Canonical Text Service and Canonical URN tool suite for Python

.. moduleauthor:: Thibault Clérice <leponteineptique@gmail.com>

"""

__name__ = "MyCapytain"
__version__ = "0.0.3"
__all__ = ["common", "endpoints", "resources"]
<|endoftext|>


<commit_before>
import run

import datetime
import numpy as np

import dataset
import neural

data = dataset.news.letters()
latent_size = 100
print('dataset build')

sutskever = neural.network.Sutskever(
    max_output_size=data.max_output_size, verbose=True, indexed_input=True
)

# Setup layers
sutskever.set_encoder_input(neural.layer.Input(data.n_classes, indexed=True))
sutskever.push_encoder_layer(neural.layer.LSTM(200))
sutskever.push_encoder_layer(neural.layer.LSTM(latent_size))

sutskever.set_decoder_input(neural.layer.Input(data.n_classes))
sutskever.push_decoder_layer(neural.layer.LSTM(latent_size))
sutskever.push_decoder_layer(neural.layer.LSTM(200))
sutskever.push_decoder_layer(neural.layer.Softmax(data.n_classes))

# Setup loss function
sutskever.set_loss(neural.loss.NaiveEntropy())
sutskever.set_optimizer(neural.optimizer.Momentum())

# Compile train, test and predict functions
sutskever.compile()

results = run.minibatch_learn(sutskever, data, test_size=1000,
                              max_time=datetime.timedelta(hours=23),
                              learning_rate=0.07, momentum=0.2)

np.savez_compressed(run.output_file + '.epoch.npz', **results)
np.savez_compressed(run.output_file + '.weights.npz', *sutskever.get_weights())

print('completed')
<commit_msg>[fix] use RMSgrave for news_train<commit_after>
import run

import datetime
import numpy as np

import dataset
import neural

data = dataset.news.letters()
latent_size = 100
print('dataset build')

sutskever = neural.network.Sutskever(
    max_output_size=data.max_output_size, verbose=True, indexed_input=True
)

# Setup layers
sutskever.set_encoder_input(neural.layer.Input(data.n_classes, indexed=True))
sutskever.push_encoder_layer(neural.layer.LSTM(200))
sutskever.push_encoder_layer(neural.layer.LSTM(latent_size))

sutskever.set_decoder_input(neural.layer.Input(data.n_classes))
sutskever.push_decoder_layer(neural.layer.LSTM(latent_size))
sutskever.push_decoder_layer(neural.layer.LSTM(200))
sutskever.push_decoder_layer(neural.layer.Softmax(data.n_classes))

# Setup loss function
sutskever.set_loss(neural.loss.NaiveEntropy())
sutskever.set_optimizer(neural.optimizer.RMSgrave())

# Compile train, test and predict functions
sutskever.compile()

results = run.minibatch_learn(sutskever, data, test_size=1000,
                              max_time=datetime.timedelta(hours=23),
                              learning_rate=0.07, momentum=0.2)

np.savez_compressed(run.output_file + '.epoch.npz', **results)
np.savez_compressed(run.output_file + '.weights.npz', *sutskever.get_weights())

print('completed')
<|endoftext|>


<commit_before># pylint: disable=W0232,R0903
from django.db import models

from platforms.models import Platform

ARCH_CHOICES = (
    ('i386', '32 bit'),
    ('x86_64', '64 bit'),
    ('arm', 'ARM'),
)

class Runner(models.Model):
    """ Model definition for the runners """
    name = models.CharField(max_length=127)
    slug = models.SlugField(unique=True)
    website = models.CharField(max_length=127, blank=True)
    icon = models.ImageField(upload_to='runners/icons', blank=True)
    platforms = models.ManyToManyField(Platform, related_name='runners')

    # pylint: disable=W0232, R0903
    class Meta(object):
        ordering = ['name']

    def __unicode__(self):
        return self.name

    def save(self, *args, **kwargs):
        return super(Runner, self).save(*args, **kwargs)

    @staticmethod
    def autocomplete_search_fields():
        return ('name__icontains', )

class RunnerVersion(models.Model):
    class Meta(object):
        ordering = ('version', 'architecture')

    def __unicode__(self):
        return u"{} v{} ({})".format(self.runner.name,
                                     self.version,
                                     self.architecture)

    runner = models.ForeignKey(Runner, related_name='versions')
    version = models.CharField(max_length=32)
    architecture = models.CharField(max_length=8,
                                    choices=ARCH_CHOICES,
                                    default='x86_64')
    url = models.URLField(blank=True)
    default = models.BooleanField(default=False)

class Runtime(models.Model):
    name = models.CharField(max_length=8)
    created_at = models.DateTimeField(auto_now=True)
    url = models.URLField()

    class Meta(object):
        ordering = ('-created_at', )

    def __unicode__(self):
        return u"{} runtime (uploaded on {})".format(self.name, self.created_at)
<commit_msg>Fix version sorting in runners<commit_after># pylint: disable=W0232,R0903
import re
from django.db import models

from platforms.models import Platform

ARCH_CHOICES = (
    ('i386', '32 bit'),
    ('x86_64', '64 bit'),
    ('arm', 'ARM'),
)

class Runner(models.Model):
    """ Model definition for the runners """
    name = models.CharField(max_length=127)
    slug = models.SlugField(unique=True)
    website = models.CharField(max_length=127, blank=True)
    icon = models.ImageField(upload_to='runners/icons', blank=True)
    platforms = models.ManyToManyField(Platform, related_name='runners')

    # pylint: disable=W0232, R0903
    class Meta(object):
        ordering = ['name']

    def __unicode__(self):
        return self.name

    def save(self, *args, **kwargs):
        return super(Runner, self).save(*args, **kwargs)

    @staticmethod
    def autocomplete_search_fields():
        return ('name__icontains', )

    @property
    def versions(self):
        def version_key(runner_version):
            version = runner_version.version
            version_match = re.search(r'(\d[\d\.]+\d)', version)
            if not version_match:
                return
            version_number = version_match.groups()[0]
            prefix = version[0:version_match.span()[0]]
            suffix = version[version_match.span()[1]:]
            version = [int(p) for p in version_number.split('.')]
            version = version + [0] * (10 - len(version))
            version.append(prefix)
            version.append(suffix)
            return version
        return sorted(self.runner_versions.all(), key=version_key)

class RunnerVersion(models.Model):
    class Meta(object):
        ordering = ('version', 'architecture')

    def __unicode__(self):
        return u"{} v{} ({})".format(self.runner.name,
                                     self.version,
                                     self.architecture)

    runner = models.ForeignKey(Runner, related_name='runner_versions')
    version = models.CharField(max_length=32)
    architecture = models.CharField(max_length=8,
                                    choices=ARCH_CHOICES,
                                    default='x86_64')
    url = models.URLField(blank=True)
    default = models.BooleanField(default=False)

class Runtime(models.Model):
    name = models.CharField(max_length=8)
    created_at = models.DateTimeField(auto_now=True)
    url = models.URLField()

    class Meta(object):
        ordering = ('-created_at', )

    def __unicode__(self):
        return u"{} runtime (uploaded on {})".format(self.name, self.created_at)
<|endoftext|>


<commit_before># Saltbot
# Copyright 2015 Adam Greig
# Licensed under the MIT license, see LICENCE file for details.

import hmac
import hashlib
import logging

from flask import Flask, request, g, jsonify, abort
from tornado.wsgi import WSGIContainer
from tornado.httpserver import HTTPServer
from tornado.ioloop import IOLoop
from tornado.netutil import bind_unix_socket
from peewee import fn, JOIN_LEFT_OUTER, SQL

try:
    from imp import reload
except ImportError:
    pass

# This cute trick ensures that if someone calls reload(webapp), we'll
# reload our own local dependencies that may also have changed.
try:
    reloading
except NameError:
    reloading = False
else:
    from . import database
    from . import serialisers
    reload(database)
    reload(serialisers)

from .database import Database
from .database import GitHubPush, SaltJob, SaltJobMinion, SaltMinionResult
from .serialisers import serialise

app = Flask(__name__)
logger = logging.getLogger("saltbot.http")

def get_page(query):
    """Given a query object, work out relevant pagination numbers"""
    page = int(request.args.get('page', 1))
    per_page = app.config['web']['per_page']
    pages = query.count() // per_page + 1
    return page, pages, per_page

def bool_and(*args, **kwargs):
    """To do BOOL_AND() etc portably we swap depending on DB engine"""
    if app.config['database']['engine'] == "sqlite":
        return fn.Min(*args, **kwargs)
    elif app.config['database']['engine'] == "postgresql":
        return fn.Bool_And(*args, **kwargs)

def bool_sum(field):
    """Count bools portably between sqlite/postgresql"""
    if app.config['database']['engine'] == "sqlite":
        return fn.Sum(SQL(field))
    elif app.config['database']['engine'] == "postgresql":
        return fn.Sum(SQL("{}::int".format(field)))

@app.before_request
def before_request():
    g._db = Database(app.config)
    g._db.connect()

@app.teardown_appcontext
def teardown_appcontext(error=None):
    if hasattr(g, '_db'):
        g._db.close()

@app.route("/api/pushes/")
def pushes():
    pushesq = GitHubPush.select().join(SaltJob).order_by(GitHubPush.id.desc())
    page, pages, pp = get_page(pushesq)
    pushes = [serialise(p) for p in pushesq.paginate(page, pp).iterator()]
    return jsonify(page=page, pages=pages, pushes=pushes)

@app.route("/api/pushes/<int:pushid>")
def push(pushid):
    try:
        push = GitHubPush.get(id=pushid)
    except GitHubPush.DoesNotExist:
        abort(404)
    return jsonify(**serialise(push))

@app.route("/api/jobs/")
def jobs():
    """
    Get all the SaltJobs we know about, and additionally:
        * Check for all_in by taking AND( COUNT(results) > 0 ).
          This is a little subtle as it's a nested aggregation so uses a
          subquery that selects everything we want plus the COUNT per-minion,
          then on the outer query performs the AND and groups by job.
        * Check for no_errors job-wide by taking AND(result) over all results.
          If some minions have no results, their AND(result) will be NULL,
          which bubbles up to a NULL at the top level that obscures whether
          any errors have actually been reported yet.
          So instead we can take the AND() again at the top level to clear
          this up.
    On top of that, in PostgreSQL (but not SQLite) all columns from the
    subquery must either appear inside an aggregate or the GROUP_BY,
    which means instead of just grouping by `id' and selecting * on the outer
    query, we must explicitly list the fields for the model. Sigh.
    """
    job_fields = SQL(
        '"id", "when", "jid", "expr_form", "target", "github_push_id"')

    no_errors_int = bool_and(SaltMinionResult.result).alias('no_errors_int')
    no_errors = bool_and(SQL('no_errors_int')).alias('no_errors')

    got_results = (fn.Count(SaltMinionResult.id) > 0).alias('got_results')
    all_in = bool_and(SQL('got_results')).alias('all_in')

    subq = (SaltJob
            .select(SaltJob, no_errors_int, got_results)
            .join(SaltJobMinion, JOIN_LEFT_OUTER)
            .join(SaltMinionResult, JOIN_LEFT_OUTER)
            .group_by(SaltJob, SaltJobMinion))

    jobsq = (SaltJob
             .select(job_fields, no_errors, all_in)
             .from_(subq.alias("subq"))
             .order_by(SQL('"id" DESC'))
             .group_by(job_fields))

    page, pages, pp = get_page(jobsq)

    jobs = [serialise(j) for j in jobsq.paginate(page, pp).iterator()]

    return jsonify(page=page, pages=pages, jobs=jobs)

@app.route("/api/jobs/<jid>")
def job(jid):
    """
    Look up one specific Job
    """
    try:
        job = SaltJob.get(jid=jid)
    except SaltJob.DoesNotExist:
        abort(404)

    no_errors = bool_and(SaltMinionResult.result).alias('no_errors')
    num_results = fn.Count(SaltMinionResult.id).alias('num_results')
    num_good = bool_sum('result').alias('num_good')

    minionsq = (SaltJobMinion
                .select(SaltJobMinion, no_errors, num_good, num_results)
                .join(SaltMinionResult, JOIN_LEFT_OUTER)
                .group_by(SaltJobMinion)
                .order_by(SaltJobMinion.id.desc())
                .where(SaltJobMinion.job == job))
    minions = [serialise(m) for m in minionsq.iterator()]

    job.no_errors = all(m['no_errors'] for m in minions)
    job.all_in = all(m['num_results'] > 0 for m in minions)

    return jsonify(minions=minions, **serialise(job))

@app.route("/api/jobs/<jid>/minions/<int:minion>")
def minionresults(jid, minion):
    """
    Look up one minion from a job, and get its results too.
    """
    try:
        job = SaltJob.get(jid=jid)
        minion = SaltJobMinion.get(id=minion, job=job)
    except (SaltJob.DoesNotExist, SaltJobMinion.DoesNotExist):
        abort(404)

    resultsq = (SaltMinionResult
                .select()
                .join(SaltJobMinion)
                .order_by(SaltMinionResult.id.desc())
                .where(SaltMinionResult.minion == minion))
    results = [serialise(r) for r in resultsq.iterator()]

    minion.no_errors = bool(all(r['result'] for r in results))

    return jsonify(results=results, **serialise(minion))

@app.route("/api/webhook", methods=["POST"])
def webhook():
    """
    Process receiving a webhook from GitHub.
    Only supports push event notifications.
    """
    logger.info("Webhook received")

    secret = app.config['github']['secret'].encode()
    their_sig = request.headers.get('X-Hub-Signature')
    my_sig = hmac.new(secret, request.data, hashlib.sha1).hexdigest()
    if not hmac.compare_digest(their_sig, "sha1={}".format(my_sig)):
        logger.warn("Invalid signature received!")
        return "Invalid signature", 403
    logger.info("HMAC signature valid")

    if request.headers.get('X-GitHub-Event') == 'push':
        event = request.get_json()
        push = {}
        push['gitref'] = event['ref']
        push['repo_name'] = event['repository']['full_name']
        push['repo_url'] = event['repository']['url']
        push['commit_id'] = event['head_commit']['id']
        push['commit_msg'] = event['head_commit']['message']
        push['commit_ts'] = event['head_commit']['timestamp']
        push['commit_url'] = event['head_commit']['url']
        push['commit_author'] = event['head_commit']['author']['username']
        push['pusher'] = event['pusher']['name']
        logger.info("Details: {}".format(push))
        app.config['webpq'].put(("github_push", push))

    if request.headers.get('X-GitHub-Event') == 'ping':
        logger.info("Received GitHub ping")

    return "OK"

def run(config, webpq):
    logger.info("App starting up")
    app.config.update(config)
    app.config['webpq'] = webpq
    server = HTTPServer(WSGIContainer(app))
    if config['web'].get('socket'):
        mode = int(str(config['web'].get('mode', 777)), 8)
        socket = bind_unix_socket(config['web'].get('socket'), mode=mode)
        server.add_socket(socket)
    else:
        port = config['web'].get('port', 8000)
        host = config['web'].get('host', 'localhost')
        server.listen(port, address=host)
    IOLoop.instance().start()
<commit_msg>old py2 doesn't have hmac.compare_digest, great<commit_after># Saltbot
# Copyright 2015 Adam Greig
# Licensed under the MIT license, see LICENCE file for details.

import hmac
import hashlib
import logging

from flask import Flask, request, g, jsonify, abort
from tornado.wsgi import WSGIContainer
from tornado.httpserver import HTTPServer
from tornado.ioloop import IOLoop
from tornado.netutil import bind_unix_socket
from peewee import fn, JOIN_LEFT_OUTER, SQL

try:
    from imp import reload
except ImportError:
    pass

# This cute trick ensures that if someone calls reload(webapp), we'll
# reload our own local dependencies that may also have changed.
try:
    reloading
except NameError:
    reloading = False
else:
    from . import database
    from . import serialisers
    reload(database)
    reload(serialisers)

from .database import Database
from .database import GitHubPush, SaltJob, SaltJobMinion, SaltMinionResult
from .serialisers import serialise

app = Flask(__name__)
logger = logging.getLogger("saltbot.http")

if not hasattr(hmac, 'compare_digest'):
    import warnings
    warnings.warn("hmac module does not have compare_digest, will use insecure"
                  " direct comparison.")

def get_page(query):
    """Given a query object, work out relevant pagination numbers"""
    page = int(request.args.get('page', 1))
    per_page = app.config['web']['per_page']
    pages = query.count() // per_page + 1
    return page, pages, per_page

def bool_and(*args, **kwargs):
    """To do BOOL_AND() etc portably we swap depending on DB engine"""
    if app.config['database']['engine'] == "sqlite":
        return fn.Min(*args, **kwargs)
    elif app.config['database']['engine'] == "postgresql":
        return fn.Bool_And(*args, **kwargs)

def bool_sum(field):
    """Count bools portably between sqlite/postgresql"""
    if app.config['database']['engine'] == "sqlite":
        return fn.Sum(SQL(field))
    elif app.config['database']['engine'] == "postgresql":
        return fn.Sum(SQL("{}::int".format(field)))

def compare_digest(a, b):
    if hasattr(hmac, 'compare_digest'):
        return hmac.compare_digest(a, b)
    else:
        return a == b

@app.before_request
def before_request():
    g._db = Database(app.config)
    g._db.connect()

@app.teardown_appcontext
def teardown_appcontext(error=None):
    if hasattr(g, '_db'):
        g._db.close()

@app.route("/api/pushes/")
def pushes():
    pushesq = GitHubPush.select().join(SaltJob).order_by(GitHubPush.id.desc())
    page, pages, pp = get_page(pushesq)
    pushes = [serialise(p) for p in pushesq.paginate(page, pp).iterator()]
    return jsonify(page=page, pages=pages, pushes=pushes)

@app.route("/api/pushes/<int:pushid>")
def push(pushid):
    try:
        push = GitHubPush.get(id=pushid)
    except GitHubPush.DoesNotExist:
        abort(404)
    return jsonify(**serialise(push))

@app.route("/api/jobs/")
def jobs():
    """
    Get all the SaltJobs we know about, and additionally:
        * Check for all_in by taking AND( COUNT(results) > 0 ).
          This is a little subtle as it's a nested aggregation so uses a
          subquery that selects everything we want plus the COUNT per-minion,
          then on the outer query performs the AND and groups by job.
        * Check for no_errors job-wide by taking AND(result) over all results.
          If some minions have no results, their AND(result) will be NULL,
          which bubbles up to a NULL at the top level that obscures whether
          any errors have actually been reported yet.
          So instead we can take the AND() again at the top level to clear
          this up.
    On top of that, in PostgreSQL (but not SQLite) all columns from the
    subquery must either appear inside an aggregate or the GROUP_BY,
    which means instead of just grouping by `id' and selecting * on the outer
    query, we must explicitly list the fields for the model. Sigh.
    """
    job_fields = SQL(
        '"id", "when", "jid", "expr_form", "target", "github_push_id"')

    no_errors_int = bool_and(SaltMinionResult.result).alias('no_errors_int')
    no_errors = bool_and(SQL('no_errors_int')).alias('no_errors')

    got_results = (fn.Count(SaltMinionResult.id) > 0).alias('got_results')
    all_in = bool_and(SQL('got_results')).alias('all_in')

    subq = (SaltJob
            .select(SaltJob, no_errors_int, got_results)
            .join(SaltJobMinion, JOIN_LEFT_OUTER)
            .join(SaltMinionResult, JOIN_LEFT_OUTER)
            .group_by(SaltJob, SaltJobMinion))

    jobsq = (SaltJob
             .select(job_fields, no_errors, all_in)
             .from_(subq.alias("subq"))
             .order_by(SQL('"id" DESC'))
             .group_by(job_fields))

    page, pages, pp = get_page(jobsq)

    jobs = [serialise(j) for j in jobsq.paginate(page, pp).iterator()]

    return jsonify(page=page, pages=pages, jobs=jobs)

@app.route("/api/jobs/<jid>")
def job(jid):
    """
    Look up one specific Job
    """
    try:
        job = SaltJob.get(jid=jid)
    except SaltJob.DoesNotExist:
        abort(404)

    no_errors = bool_and(SaltMinionResult.result).alias('no_errors')
    num_results = fn.Count(SaltMinionResult.id).alias('num_results')
    num_good = bool_sum('result').alias('num_good')

    minionsq = (SaltJobMinion
                .select(SaltJobMinion, no_errors, num_good, num_results)
                .join(SaltMinionResult, JOIN_LEFT_OUTER)
                .group_by(SaltJobMinion)
                .order_by(SaltJobMinion.id.desc())
                .where(SaltJobMinion.job == job))
    minions = [serialise(m) for m in minionsq.iterator()]

    job.no_errors = all(m['no_errors'] for m in minions)
    job.all_in = all(m['num_results'] > 0 for m in minions)

    return jsonify(minions=minions, **serialise(job))

@app.route("/api/jobs/<jid>/minions/<int:minion>")
def minionresults(jid, minion):
    """
    Look up one minion from a job, and get its results too.
    """
    try:
        job = SaltJob.get(jid=jid)
        minion = SaltJobMinion.get(id=minion, job=job)
    except (SaltJob.DoesNotExist, SaltJobMinion.DoesNotExist):
        abort(404)

    resultsq = (SaltMinionResult
                .select()
                .join(SaltJobMinion)
                .order_by(SaltMinionResult.id.desc())
                .where(SaltMinionResult.minion == minion))
    results = [serialise(r) for r in resultsq.iterator()]

    minion.no_errors = bool(all(r['result'] for r in results))

    return jsonify(results=results, **serialise(minion))

@app.route("/api/webhook", methods=["POST"])
def webhook():
    """
    Process receiving a webhook from GitHub.
    Only supports push event notifications.
    """
    logger.info("Webhook received")

    secret = app.config['github']['secret'].encode()
    their_sig = request.headers.get('X-Hub-Signature')
    my_sig = hmac.new(secret, request.data, hashlib.sha1).hexdigest()
    if not compare_digest(their_sig, "sha1={}".format(my_sig)):
        logger.warn("Invalid signature received!")
        return "Invalid signature", 403
    logger.info("HMAC signature valid")

    if request.headers.get('X-GitHub-Event') == 'push':
        event = request.get_json()
        push = {}
        push['gitref'] = event['ref']
        push['repo_name'] = event['repository']['full_name']
        push['repo_url'] = event['repository']['url']
        push['commit_id'] = event['head_commit']['id']
        push['commit_msg'] = event['head_commit']['message']
        push['commit_ts'] = event['head_commit']['timestamp']
        push['commit_url'] = event['head_commit']['url']
        push['commit_author'] = event['head_commit']['author']['username']
        push['pusher'] = event['pusher']['name']
        logger.info("Details: {}".format(push))
        app.config['webpq'].put(("github_push", push))

    if request.headers.get('X-GitHub-Event') == 'ping':
        logger.info("Received GitHub ping")

    return "OK"

def run(config, webpq):
    logger.info("App starting up")
    app.config.update(config)
    app.config['webpq'] = webpq
    server = HTTPServer(WSGIContainer(app))
    if config['web'].get('socket'):
        mode = int(str(config['web'].get('mode', 777)), 8)
        socket = bind_unix_socket(config['web'].get('socket'), mode=mode)
        server.add_socket(socket)
    else:
        port = config['web'].get('port', 8000)
        host = config['web'].get('host', 'localhost')
        server.listen(port, address=host)
    IOLoop.instance().start()
<|endoftext|>