<commit_before>#!/usr/bin/python
# <variable> = required
# [variable] = optional
# Usage ./lava-job-runner.py <username> <token> <lava_server_url> [job_repo] [bundle_stream]

import xmlrpclib
import subprocess
import fnmatch
import os
import json
import time
import re
import urlparse
import argparse

job_map = {}

def report(jobs):
    for job in jobs:
        print str(job) + ' ' + jobs[job]

def poll_jobs(connection):
    run = True
    submitted_jobs = {}
    finished_jobs = {}

    for job in job_map:
        if job_map[job] is not None:
            # Push
            submitted_jobs[job_map[job]] = job

    while run:
        if not submitted_jobs:
            run = False
            break
        for job in submitted_jobs:
            try:
                status = connection.scheduler.job_status(job)
                if status['job_status'] == 'Complete':
                    print 'job-id-' + str(job) + '-' + os.path.basename(submitted_jobs[job]) + ' : pass'
                    # Pop
                    finished_jobs[job] = 'PASS'
                    submitted_jobs.pop(job, None)
                    break
                elif status['job_status'] == 'Incomplete' or status['job_status'] == 'Canceled' or status['job_status'] == 'Canceling':
                    print 'job-id-' + str(job) + '-' + os.path.basename(submitted_jobs[job]) + ' : fail'
                    # Pop
                    finished_jobs[job] = 'FAIL'
                    submitted_jobs.pop(job, None)
                    break
                else:
                    print str(job) + ' - ' + str(status['job_status'])
                    time.sleep(10)
            except (xmlrpclib.ProtocolError, xmlrpclib.Fault, IOError) as e:
                print "POLLING ERROR!"
                print e
                continue

    return finished_jobs

def submit_jobs(connection, server, bundle_stream):
    online_devices, offline_devices = gather_devices(connection)
    online_device_types, offline_device_types = gather_device_types(connection)
    print "Submitting Jobs to Server..."
    for job in job_map:
        try:
            with open(job, 'rb') as stream:
                job_data = stream.read()
                # Injection
                if bundle_stream is not None:
                    job_data = re.sub('LAVA_SERVER', server, job_data)
                    job_data = re.sub('BUNDLE_STREAM', bundle_stream, job_data)
            job_info = json.loads(job_data)
            # Check if request device(s) are available
            if 'target' in job_info:
                if job_info['target'] in offline_devices:
                    print "%s is OFFLINE skipping submission" % job_info['target']
                    print os.path.basename(job) + ': skip'
                elif job_info['target'] in online_devices:
                    pass
                    job_map[job] = connection.scheduler.submit_job(job_data)
                else:
                    print "No target available on server, skipping..."
                    print os.path.basename(job) + ' : skip'
            elif 'device_type' in job_info:
                if job_info['device_type'] in offline_device_types:
                    print "All device types: %s are OFFLINE, skipping..." % job_info['device_type']
                    print os.path.basename(job) + ' : skip'
                elif job_info['device_type'] in online_device_types:
                    pass
                    job_map[job] = connection.scheduler.submit_job(job_data)
                else:
                    print "No device-type available on server, skipping..."
                    print os.path.basename(job) + ' : skip'
            elif 'device_group' in job_info:
                print "Multinode Job Detected! Checking if required devices are available..."
                multinode_online_device_types = online_device_types
                server_has_required_devices = True
                for groups in job_info['device_group']:
                    if groups['device_type'] in offline_device_types:
                        print "All device types: %s are OFFLINE, skipping..." % groups['device_type']
                        server_has_required_devices = False
                        print os.path.basename(job) + ' : skip'
                        break
                    elif groups['device_type'] in online_device_types:
                        if groups['count'] > multinode_online_device_types[groups['device_type']]:
                            print "Server does not have enough online devices to submit job!"
                            print os.path.basename(job) + ' : skip'
                            server_has_required_devices = False
                            break
                        elif groups['count'] <= multinode_online_device_types[groups['device_type']]:
                            print "Server has enough devices for this group!"
                            multinode_online_device_types[groups['device_type']] = multinode_online_device_types[groups['device_type']] - groups['count']
                        else:
                            print "Should never get here!"
                            print os.path.basename(job) + ' : skip'
                            server_has_required_devices = False
                            break
                    else:
                        print "No device-type available on server, skipping..."
                        print os.path.basename(job) + ' : skip'
                if server_has_required_devices:
                    print "Submitting Multinode Job!"
                    job_map[job] = connection.scheduler.submit_job(job_data)[0]
            else:
                print "Should never get here"
                print os.path.basename(job) + ' : skip'
        except (xmlrpclib.ProtocolError, xmlrpclib.Fault, IOError, ValueError) as e:
            print "JSON VALIDATION ERROR!"
            print job
            print e
            continue

def load_jobs():
    top = os.getcwd()
    for root, dirnames, filenames in os.walk(top):
        for filename in fnmatch.filter(filenames, '*.json'):
            job_map[os.path.join(root, filename)] = None

def retrieve_jobs(jobs):
    cmd = 'git clone %s' % jobs
    try:
        print "Cloning LAVA Jobs..."
        subprocess.check_output(cmd, shell=True)
        print "Clone Successful!"
        print "clone-jobs : pass"
    except subprocess.CalledProcessError as e:
        print "ERROR!"
        print "Unable to clone %s" % jobs
        print "clone-jobs : fail"
        exit(1)

def gather_devices(connection):
    online_devices = {}
    offline_devices = {}
    print "Gathering Devices..."
    all_devices = connection.scheduler.all_devices()
    for device in all_devices:
        if device[2] == 'offline':
            offline_devices[device[0]] = 1
        else:
            online_devices[device[0]] = 1
    print "Gathered Devices Successfully!"
    return online_devices, offline_devices

def gather_device_types(connection):
    online_device_types = {}
    offline_device_types = {}
    print "Gathering Device Types..."
    all_device_types = connection.scheduler.all_device_types()
    for device_type in all_device_types:
        # Only use dictionary data structures
        if isinstance(device_type, dict):
            # Retired
            if device_type['idle'] == 0 and device_type['busy'] == 0 and device_type['offline'] == 0:
                offline_device_types[device_type['name']] = 0
            # Running
            elif device_type['idle'] > 0 or device_type['busy'] > 0:
                online_device_types[device_type['name']] = device_type['idle'] + device_type['busy']
            # Offline
            else:
                offline_device_types[device_type['name']] = device_type['offline']
    print "Gathered Device Types Successfully!"
    return online_device_types, offline_device_types

def connect(url):
    try:
        print "Connecting to Server..."
        connection = xmlrpclib.ServerProxy(url)

        print "Connection Successful!"
        print "connect-to-server : pass"
        return connection
    except (xmlrpclib.ProtocolError, xmlrpclib.Fault, IOError) as e:
        print "CONNECTION ERROR!"
        print "Unable to connect to %s" % url
        print e
        print "connect-to-server : fail"
        exit(1)

def validate_input(username, token, server):
    url = urlparse.urlparse(server)
    if url.path.find('RPC2') == -1:
        print "LAVA Server URL must end with /RPC2"
        exit(1)
    return url.scheme + '://' + username + ':' + token + '@' + url.netloc + url.path

def main(args):

    url = validate_input(args.username, args.token, args.server)
    connection = connect(url)
    if args.repo:
        retrieve_jobs(args.repo)
    load_jobs()
    if args.stream:
        submit_jobs(connection, args.server, args.stream)
    else:
        submit_jobs(connection, args.server, bundle_stream=None)
    if args.poll:
        jobs = poll_jobs(connection)
        if args.report:
            report(jobs)
    exit(0)

if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument("username", help="username for the LAVA server")
    parser.add_argument("token", help="token for LAVA server api")
    parser.add_argument("server", help="server url for LAVA server")
    parser.add_argument("--stream", help="bundle stream for LAVA server")
    parser.add_argument("--repo", help="git repo for LAVA jobs")
    parser.add_argument("--poll", action='store_true', help="poll the submitted LAVA jobs")
    parser.add_argument("--report", action='store_true', help="report on the submitted LAVA jobs")
    args = parser.parse_args()
    main(args)<commit_msg>kernel-ci: implement --kreport<commit_after>#!/usr/bin/python
# <variable> = required
# [variable] = optional
# Usage ./lava-job-runner.py <username> <token> <lava_server_url> [job_repo] [bundle_stream]

import xmlrpclib
import subprocess
import fnmatch
import os
import json
import time
import re
import urlparse
import argparse
import shutil

job_map = {}

def kreport(connection, jobs, duration):
    results_directory = os.getcwd() + '/results'
    results = {}
    mkdir(results_directory)
    for job_id in jobs:
        # Retrieve job details
        job_details = connection.scheduler.job_details(job_id)
        if job_details['requested_device_type_id']:
            device_type = job_details['requested_device_type_id']
        if job_details['description']:
            job_name = job_details['description']
        result = jobs[job_id]['result']
        bundle = jobs[job_id]['bundle']
        # Retrieve the log file
        binary_job_file = connection.scheduler.job_output(job_id)
        job_file = str(binary_job_file)
        # Retrieve bundle
        json_bundle = connection.dashboard.get(bundle)
        bundle_data = json.loads(json_bundle['content'])
        if bundle_data['test_runs'][0]['attributes']['kernel.defconfig']:
            defconfig = bundle_data['test_runs'][0]['attributes']['kernel.defconfig']
        if bundle_data['test_runs'][0]['attributes']['kernel.version']:
            kernel_version = bundle_data['test_runs'][0]['attributes']['kernel.version']
        if bundle_data['test_runs'][0]['attributes']['kernel-boot-time']:
            kernel_boot_time = bundle_data['test_runs'][0]['attributes']['kernel-boot-time']
        if bundle_data['test_runs'][0]['attributes']['kernel.tree']:
            kernel_tree = bundle_data['test_runs'][0]['attributes']['kernel.tree']
        # Record the results
        # TODO: Will need to map device_types to dashboard device types
        if defconfig and device_type and kernel_boot_time and result:
            log = 'boot-%s.log' % device_type
            directory = os.path.join(results_directory, defconfig)
            mkdir(directory)
            write_file(job_file, log, directory)
            if results.has_key(defconfig):
                results[defconfig].append({'device_type': device_type, 'kernel_boot_time': kernel_boot_time, 'result': result})
            else:
                results[defconfig] = [{'device_type': device_type, 'kernel_boot_time': kernel_boot_time, 'result': result}]
    if results and kernel_tree and kernel_version:
        log = '%s-boot-report.log' % kernel_version
        with open(os.path.join(results_directory, log), 'a') as f:
            f.write('Status Dashboard: http://status.armcloud.us/boot/all/job/%s/kernel/%s/\n' % (kernel_tree, kernel_version))
            f.write('\n')
            f.write('Total duration: %.2f seconds\n' % duration)
            f.write('Tree/Branch: %s\n' % kernel_tree)
            f.write('Git describe: %s\n' % kernel_version)
            f.write('\n')
            f.write('Full Report:\n')
            for defconfig, results_list in results.items():
                f.write('\n')
                f.write(defconfig)
                f.write('\n')
                for result in results_list:
                    f.write('    %s   %ss   %s\n' % (result['device_type'], result['kernel_boot_time'], result['result']))

def write_file(file, name, directory):
    with open(os.path.join(directory, name), 'w') as f:
        f.write(file)

def mkdir(directory):
    if not os.path.exists(directory):
        os.makedirs(directory)
    else:
        shutil.rmtree(directory)
        os.makedirs(directory)

def poll_jobs(connection):
    run = True
    submitted_jobs = {}
    finished_jobs = {}

    for job in job_map:
        if job_map[job] is not None:
            # Push
            submitted_jobs[job_map[job]] = job

    while run:
        if not submitted_jobs:
            run = False
            break
        for job in submitted_jobs:
            try:
                status = connection.scheduler.job_status(job)
                if status['job_status'] == 'Complete':
                    print 'job-id-' + str(job) + '-' + os.path.basename(submitted_jobs[job]) + ' : pass'
                    # Pop
                    if status['bundle_sha1']:
                        finished_jobs[job] = {'result': 'PASS', 'bundle': status['bundle_sha1']}
                    else:
                        finished_jobs[job] = {'result': 'PASS', 'bundle': None}
                    submitted_jobs.pop(job, None)
                    break
                elif status['job_status'] == 'Incomplete' or status['job_status'] == 'Canceled' or status['job_status'] == 'Canceling':
                    print 'job-id-' + str(job) + '-' + os.path.basename(submitted_jobs[job]) + ' : fail'
                    # Pop
                    if status['bundle_sha1']:
                        finished_jobs[job] = {'result': 'FAIL', 'bundle': status['bundle_sha1']}
                    else:
                        finished_jobs[job] = {'result': 'FAIL', 'bundle': None}
                    submitted_jobs.pop(job, None)
                    break
                else:
                    print str(job) + ' - ' + str(status['job_status'])
                    time.sleep(10)
            except (xmlrpclib.ProtocolError, xmlrpclib.Fault, IOError) as e:
                print "POLLING ERROR!"
                print e
                continue

    return finished_jobs

def submit_jobs(connection, server, bundle_stream):
    online_devices, offline_devices = gather_devices(connection)
    online_device_types, offline_device_types = gather_device_types(connection)
    print "Submitting Jobs to Server..."
    for job in job_map:
        try:
            with open(job, 'rb') as stream:
                job_data = stream.read()
                # Injection
                if bundle_stream is not None:
                    job_data = re.sub('LAVA_SERVER', server, job_data)
                    job_data = re.sub('BUNDLE_STREAM', bundle_stream, job_data)
            job_info = json.loads(job_data)
            # Check if request device(s) are available
            if 'target' in job_info:
                if job_info['target'] in offline_devices:
                    print "%s is OFFLINE skipping submission" % job_info['target']
                    print os.path.basename(job) + ': skip'
                elif job_info['target'] in online_devices:
                    pass
                    job_map[job] = connection.scheduler.submit_job(job_data)
                else:
                    print "No target available on server, skipping..."
                    print os.path.basename(job) + ' : skip'
            elif 'device_type' in job_info:
                if job_info['device_type'] in offline_device_types:
                    print "All device types: %s are OFFLINE, skipping..." % job_info['device_type']
                    print os.path.basename(job) + ' : skip'
                elif job_info['device_type'] in online_device_types:
                    pass
                    job_map[job] = connection.scheduler.submit_job(job_data)
                else:
                    print "No device-type available on server, skipping..."
                    print os.path.basename(job) + ' : skip'
            elif 'device_group' in job_info:
                print "Multinode Job Detected! Checking if required devices are available..."
                multinode_online_device_types = online_device_types
                server_has_required_devices = True
                for groups in job_info['device_group']:
                    if groups['device_type'] in offline_device_types:
                        print "All device types: %s are OFFLINE, skipping..." % groups['device_type']
                        server_has_required_devices = False
                        print os.path.basename(job) + ' : skip'
                        break
                    elif groups['device_type'] in online_device_types:
                        if groups['count'] > multinode_online_device_types[groups['device_type']]:
                            print "Server does not have enough online devices to submit job!"
                            print os.path.basename(job) + ' : skip'
                            server_has_required_devices = False
                            break
                        elif groups['count'] <= multinode_online_device_types[groups['device_type']]:
                            print "Server has enough devices for this group!"
                            multinode_online_device_types[groups['device_type']] = multinode_online_device_types[groups['device_type']] - groups['count']
                        else:
                            print "Should never get here!"
                            print os.path.basename(job) + ' : skip'
                            server_has_required_devices = False
                            break
                    else:
                        print "No device-type available on server, skipping..."
                        print os.path.basename(job) + ' : skip'
                if server_has_required_devices:
                    print "Submitting Multinode Job!"
                    job_map[job] = connection.scheduler.submit_job(job_data)[0]
            else:
                print "Should never get here"
                print os.path.basename(job) + ' : skip'
        except (xmlrpclib.ProtocolError, xmlrpclib.Fault, IOError, ValueError) as e:
            print "JSON VALIDATION ERROR!"
            print job
            print e
            continue

def load_jobs():
    top = os.getcwd()
    for root, dirnames, filenames in os.walk(top):
        for filename in fnmatch.filter(filenames, '*.json'):
            job_map[os.path.join(root, filename)] = None

def retrieve_jobs(jobs):
    cmd = 'git clone %s' % jobs
    try:
        print "Cloning LAVA Jobs..."
        subprocess.check_output(cmd, shell=True)
        print "Clone Successful!"
        print "clone-jobs : pass"
    except subprocess.CalledProcessError as e:
        print "ERROR!"
        print "Unable to clone %s" % jobs
        print "clone-jobs : fail"
        exit(1)

def gather_devices(connection):
    online_devices = {}
    offline_devices = {}
    print "Gathering Devices..."
    all_devices = connection.scheduler.all_devices()
    for device in all_devices:
        if device[2] == 'offline':
            offline_devices[device[0]] = 1
        else:
            online_devices[device[0]] = 1
    print "Gathered Devices Successfully!"
    return online_devices, offline_devices

def gather_device_types(connection):
    online_device_types = {}
    offline_device_types = {}
    print "Gathering Device Types..."
    all_device_types = connection.scheduler.all_device_types()
    for device_type in all_device_types:
        # Only use dictionary data structures
        if isinstance(device_type, dict):
            # Retired
            if device_type['idle'] == 0 and device_type['busy'] == 0 and device_type['offline'] == 0:
                offline_device_types[device_type['name']] = 0
            # Running
            elif device_type['idle'] > 0 or device_type['busy'] > 0:
                online_device_types[device_type['name']] = device_type['idle'] + device_type['busy']
            # Offline
            else:
                offline_device_types[device_type['name']] = device_type['offline']
    print "Gathered Device Types Successfully!"
    return online_device_types, offline_device_types

def connect(url):
    try:
        print "Connecting to Server..."
        connection = xmlrpclib.ServerProxy(url)

        print "Connection Successful!"
        print "connect-to-server : pass"
        return connection
    except (xmlrpclib.ProtocolError, xmlrpclib.Fault, IOError) as e:
        print "CONNECTION ERROR!"
        print "Unable to connect to %s" % url
        print e
        print "connect-to-server : fail"
        exit(1)

def validate_input(username, token, server):
    url = urlparse.urlparse(server)
    if url.path.find('RPC2') == -1:
        print "LAVA Server URL must end with /RPC2"
        exit(1)
    return url.scheme + '://' + username + ':' + token + '@' + url.netloc + url.path

def main(args):

    url = validate_input(args.username, args.token, args.server)
    connection = connect(url)
    if args.repo:
        retrieve_jobs(args.repo)
    load_jobs()
    start_time = time.time()
    if args.stream:
        submit_jobs(connection, args.server, args.stream)
    else:
        submit_jobs(connection, args.server, bundle_stream=None)
    if args.poll:
        jobs = poll_jobs(connection)
        end_time = time.time()
        duration = end_time - start_time
        if args.kreport:
            kreport(connection, jobs, duration)
    exit(0)

if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument("username", help="username for the LAVA server")
    parser.add_argument("token", help="token for LAVA server api")
    parser.add_argument("server", help="server url for LAVA server")
    parser.add_argument("--stream", help="bundle stream for LAVA server")
    parser.add_argument("--repo", help="git repo for LAVA jobs")
    parser.add_argument("--poll", action='store_true', help="poll the submitted LAVA jobs")
    parser.add_argument("--kreport", action='store_true', help="kernel-ci report for the submitted LAVA jobs")
    args = parser.parse_args()
    main(args)<|endoftext|>


<commit_before>import time
import sys

from pingo.parts import serial

DEFAULT_PORT = '/dev/ttyAMA0'  # the mini-UART port on the Raspberry Pi

def main(port=DEFAULT_PORT):
    lcd = serial.LCD16x2(port)
    time.sleep(.5)  # wait for display to boot up
    lcd.clear()

    while True:
        lcd.set_cursor(0, 0)
        lcd.write(time.strftime('%H:%M:%S'))
        time.sleep(1)

if __name__ == '__main__':
    main(*sys.argv[1:])
<commit_msg>Update clock.py<commit_after>import time
import sys

from pingo.parts import serial

DEFAULT_PORT = '/dev/ttyAMA0'  # the mini-UART port on the Raspberry Pi

def main(port=DEFAULT_PORT):
    lcd = serial.LCD16x2(port)
    time.sleep(.5)  # wait for display to boot up
    lcd.clear()

    while True:
        lcd.set_cursor(0, 0)
        lcd.write(time.strftime('%H:%M:%S:%f'))
        time.sleep(.1)

if __name__ == '__main__':
    main(*sys.argv[1:])
<|endoftext|>


<commit_before>from django.contrib.auth.models import User
from django.shortcuts import render_to_response
from django.contrib.auth.decorators import login_required
from django.views.decorators.csrf import csrf_exempt
from django.template import RequestContext
from django.shortcuts import redirect
from django.conf import settings

from annoying.decorators import render_to, ajax_request

from accounts.models import *

from common.admin import email_templates, utils
from common.view_helpers import _template_values, JSONResponse

#from data_stream.query_managers import *

@render_to('common/home.html')
def home(request):
    if not request.user.is_authenticated():
        return _template_values(request, page_title="home", navbar='nav_home')
    return redirect('/accounts/profile')
    filter = request.GET.get('filter', 'following')

@login_required
@csrf_exempt
@ajax_request
def feedback(request):
    """
    Endpoint to submit feedback
    """
    feedback = request.POST.get('feedback', None)
    if not feedback:
        return JSONResponse({'res':'failed'})

    feedback.replace('\n', '
')
    user = request.user
    subject = email_templates.feedback['subject']
    content = email_templates.feedback['content'] % (user.username, feedback)
    admin_emails = [admin[1] for admin in settings.ADMINS]
    utils.send_mail(subject, content, admin_emails, from_email=user.email)
    return {'res':'success'}<commit_msg>append slash<commit_after>from django.contrib.auth.models import User
from django.shortcuts import render_to_response
from django.contrib.auth.decorators import login_required
from django.views.decorators.csrf import csrf_exempt
from django.template import RequestContext
from django.shortcuts import redirect
from django.conf import settings

from annoying.decorators import render_to, ajax_request

from accounts.models import *

from common.admin import email_templates, utils
from common.view_helpers import _template_values, JSONResponse

#from data_stream.query_managers import *

@render_to('common/home.html')
def home(request):
    if not request.user.is_authenticated():
        return _template_values(request, page_title="home", navbar='nav_home')
    return redirect('/accounts/profile/')
    filter = request.GET.get('filter', 'following')

@login_required
@csrf_exempt
@ajax_request
def feedback(request):
    """
    Endpoint to submit feedback
    """
    feedback = request.POST.get('feedback', None)
    if not feedback:
        return JSONResponse({'res':'failed'})

    feedback.replace('\n', '
')
    user = request.user
    subject = email_templates.feedback['subject']
    content = email_templates.feedback['content'] % (user.username, feedback)
    admin_emails = [admin[1] for admin in settings.ADMINS]
    utils.send_mail(subject, content, admin_emails, from_email=user.email)
    return {'res':'success'}<|endoftext|>


<commit_before>#!/usr/bin/env python

from time import sleep, time
from fabric.api import execute, task, env

import app_config
import sys
import traceback

def safe_execute(*args, **kwargs):
    """
    Wrap execute() so that all exceptions are caught and logged.
    """
    try:
        execute(*args, **kwargs)
    except:
        print "ERROR [timestamp: %d]: Here's the traceback" % time()
        ex_type, ex, tb = sys.exc_info()
        traceback.print_tb(tb)
        del tb

@task
def deploy_liveblog():
    """
    Get and update liveblog slides
    """
    while True:
        start = time()
        safe_execute('liveblog.update')
        safe_execute('deploy_liveblog_slides')
        duration = int(time() - start)
        wait = app_config.LIVEBLOG_DEPLOY_INTERVAL - duration

        if wait < 0:
            print "== WARN: Deploying slides took %d seconds longer than %d ==" % (abs(wait), app_config.LIVEBLOG_DEPLOY_INTERVAL)
            wait = 0
        else:
            print "== Deploying slides ran in %ds, waiting %ds ==" % (duration, wait)

        sleep(wait)

@task
def deploy_results():
    """
    Harvest data and deploy slides indefinitely
    """
    while True:
        start = time()
        safe_execute('ap.update')
        safe_execute('render.less')
        safe_execute('deploy_bop')
        safe_execute('deploy_results_slides')
        safe_execute('deploy_big_boards')
        safe_execute('deploy_state_slides')

        duration = int(time() - start)
        wait = app_config.RESULTS_DEPLOY_INTERVAL - duration

        if wait < 0:
            print "WARN: Deploying slides took %ds longer than %ds" % (abs(wait), app_config.RESULTS_DEPLOY_INTERVAL)
            wait = 0
        else:
            print "== Deploying slides ran in %ds, waiting %ds ==" % (duration, wait)

        import sys
        sys.exit()
        sleep(wait)
<commit_msg>don't kill daemon after one cycle, delay results daemon a bit<commit_after>#!/usr/bin/env python

from time import sleep, time
from fabric.api import execute, task, env

import app_config
import sys
import traceback

def safe_execute(*args, **kwargs):
    """
    Wrap execute() so that all exceptions are caught and logged.
    """
    try:
        execute(*args, **kwargs)
    except:
        print "ERROR [timestamp: %d]: Here's the traceback" % time()
        ex_type, ex, tb = sys.exc_info()
        traceback.print_tb(tb)
        del tb

@task
def deploy_liveblog():
    """
    Get and update liveblog slides
    """
    while True:
        start = time()
        safe_execute('liveblog.update')
        safe_execute('deploy_liveblog_slides')
        duration = int(time() - start)
        wait = app_config.LIVEBLOG_DEPLOY_INTERVAL - duration

        if wait < 0:
            print "== WARN: Deploying slides took %d seconds longer than %d ==" % (abs(wait), app_config.LIVEBLOG_DEPLOY_INTERVAL)
            wait = 0
        else:
            print "== Deploying slides ran in %ds, waiting %ds ==" % (duration, wait)

        sleep(wait)

@task
def deploy_results():
    """
    Harvest data and deploy slides indefinitely
    """
    sleep(5)
    while True:
        start = time()
        safe_execute('ap.update')
        safe_execute('render.less')
        safe_execute('deploy_bop')
        safe_execute('deploy_results_slides')
        safe_execute('deploy_big_boards')
        safe_execute('deploy_state_slides')

        duration = int(time() - start)
        wait = app_config.RESULTS_DEPLOY_INTERVAL - duration

        if wait < 0:
            print "WARN: Deploying slides took %ds longer than %ds" % (abs(wait), app_config.RESULTS_DEPLOY_INTERVAL)
            wait = 0
        else:
            print "== Deploying slides ran in %ds, waiting %ds ==" % (duration, wait)

        sleep(wait)
<|endoftext|>


<commit_before>from fabric.api import task, local
import sys
import platform
import os

def is_ubuntu():
    return platform.dist()[0] == 'Ubuntu'

def is_centos():
    (centos, version, release) = platform.dist()
    if version != "6.4":
        print "Warning: centos %s is not tested" % version
    return centos == "centos"

@task
def deploy():
    """deploys the system on ubuntu"""
    # download()
    ubuntu()
    install()

@task
def download():
    '''downloads cloudmesh'''
    local("git clone git@github.com:cloudmesh/cloudmesh.git")

@task
def install():
    local("pip install -r requirements.txt")
    local("pip setup.py install")

def install_mongodb():
    if is_ubuntu():
        local('sudo apt-key adv --keyserver keyserver.ubuntu.com --recv 7F0CEB10')
        local(
              'sudo sh -c "echo \'deb http://downloads-distro.mongodb.org/repo/ubuntu-upstart dist 10gen\' > /etc/apt/sources.list.d/10gen.list"')
        local('sudo apt-get update')
        local('sudo apt-get install mongodb-10gen')
    elif is_centos():
        local("sh -c \"echo '[10gen]\nname=10gen Repository\nbaseurl=http://downloads-distro.mongodb.org/repo/redhat/os/x86_64\ngpgcheck=0\nenabled=1' >/etc/yum.repos.d/10gen\"")
        install_packages(["mongo-10gen", "mongo-10gen-server"])

    elif sys.platform == "darwin":
        local('ruby -e "$(curl -fsSL https://raw.github.com/mxcl/homebrew/go)"')
        local('brew update')
        local('brew install mongodb')

def install_package(package):
    if is_ubuntu():
        local ("sudo apt-get install {0}".format(package))
    if is_centos():
        local("sudo yum -y install {0}".format(package))
    elif sys.platform == "darwin":
        print "Not yet supported"
        sys.exit()
    elif sys.platform == "win32":
        print "Windows is not supported"
        print "Use Linux instead"
        sys.exit()

@task
def install_packages(packages):
    for package in packages:
        install_package (package)

@task
def ubuntu():
    '''prepares an system and installs all
    needed packages before we install cloudmesch'''

    install_packages(["git",
                      "curl",
                      "python-virtualenv",
                      "python-dev",
                      "libldap2-dev",
                      "libsasl2-dev"])
    install_mongodb()
    install_packages(["rabbitmq-server"])

def centos():
    install_packages ["git", "openldap-devel", "bzip2-devel", "rabbitmq-server"])
    local('sudo sh -c "chkconfig rabbitmq-server on && service rabbitmq-server start"')
    if not os.path.exists("/opt/python"):
        local('sudo mkdir -p /opt/python')
        local('sh -c "cd /tmp && wget http://www.python.org/ftp/python/2.7.5/Python-2.7.5.tgz && tar xzf python/2.7.5/Python-2.7.5.tgz && ./configure --prefix=/opt/python && make && sudo make install')
        local('sh -c "cd /tmp && curl -O https://pypi.python.org/packages/source/v/virtualenv/virtualenv-1.10.1.tar.gz && tar xfz virtualenv-1.10.1.tar.gz && virtualenv-1.10.1.tar.gz && sudo python setup.py install"')
        print "Add /opt/python/bin to your PATH"
    else:
        print "Could not install python, /opt/python already exists"
        sys.exit()
<commit_msg>Changes for centos, not working yet<commit_after>from fabric.api import task, local
import sys
import platform
import os

def is_ubuntu():
    return platform.dist()[0] == 'Ubuntu'

def is_centos():
    (centos, version, release) = platform.dist()
    if version != "6.4":
        print "Warning: centos %s is not tested" % version
    return centos == "centos"

@task
def deploy():
    """deploys the system on supported distributions"""
    # download()
    if is_ubuntu():
        ubuntu()
    elif is_centos():
        centos()
    else:
        print "OS distribution not supported; please see documatation for manual installation instructions."
        sys.exit()

    install()

@task
def download():
    '''downloads cloudmesh'''
    local("git clone git@github.com:cloudmesh/cloudmesh.git")

@task
def install():
    local("pip install -r requirements.txt")
    local("pip setup.py install")

def install_mongodb():
    if is_ubuntu():
        local('sudo apt-key adv --keyserver keyserver.ubuntu.com --recv 7F0CEB10')
        local(
              'sudo sh -c "echo \'deb http://downloads-distro.mongodb.org/repo/ubuntu-upstart dist 10gen\' > /etc/apt/sources.list.d/10gen.list"')
        local('sudo apt-get update')
        local('sudo apt-get install mongodb-10gen')
    elif is_centos():
        local("sh -c \"echo '[10gen]\nname=10gen Repository\nbaseurl=http://downloads-distro.mongodb.org/repo/redhat/os/x86_64\ngpgcheck=0\nenabled=1' >/etc/yum.repos.d/10gen\"")
        install_packages(["mongo-10gen", "mongo-10gen-server"])
    elif sys.platform == "darwin":
        local('ruby -e "$(curl -fsSL https://raw.github.com/mxcl/homebrew/go)"')
        local('brew update')
        local('brew install mongodb')

def install_package(package):
    if is_ubuntu():
        local ("sudo apt-get install {0}".format(package))
    if is_centos():
        local("sudo yum -y install {0}".format(package))
    elif sys.platform == "darwin":
        print "Not yet supported"
        sys.exit()
    elif sys.platform == "win32":
        print "Windows is not supported"
        print "Use Linux instead"
        sys.exit()

@task
def install_packages(packages):
    for package in packages:
        install_package (package)

@task
def ubuntu():
    '''prepares an system and installs all
    needed packages before we install cloudmesch'''

    install_packages(["git",
                      "curl",
                      "python-virtualenv",
                      "python-dev",
                      "libldap2-dev",
                      "libsasl2-dev"])
    install_mongodb()
    install_packages(["rabbitmq-server"])

def centos():
    install_packages ["git", "openldap-devel", "bzip2-devel", "rabbitmq-server"])
    local('sudo sh -c "chkconfig rabbitmq-server on && service rabbitmq-server start"')
    if not os.path.exists("/opt/python"):
        local('sudo mkdir -p /opt/python')
        local('sh -c "cd /tmp && wget http://www.python.org/ftp/python/2.7.5/Python-2.7.5.tgz && tar xzf python/2.7.5/Python-2.7.5.tgz && ./configure --prefix=/opt/python && make && sudo make install')
        local('sh -c "cd /tmp && curl -O https://pypi.python.org/packages/source/v/virtualenv/virtualenv-1.10.1.tar.gz && tar xfz virtualenv-1.10.1.tar.gz && virtualenv-1.10.1.tar.gz && sudo python setup.py install"')
        print "Add /opt/python/bin to your PATH"
    else:
        print "Could not install python, /opt/python already exists"
        sys.exit()
<|endoftext|>


<commit_before>"""
fab deployment script
====================================

"""
import os

from fabric.api import *
import fabric.colors

from fab_deploy import *

def my_site():
	""" Default Configuration """
	env.conf = dict(
		PROVIDER = 'ec2_us_east', # Also can use ec2_us_west
		AWS_ACCESS_KEY_ID     = '',
		AWS_SECRET_ACCESS_KEY = '',

		#PROVIDER = 'rackspace',
		#RACKSPACE_USER = '',
		#RACKSPACE_KEY  = '',

		CONF_FILE = os.path.join(os.getcwd(),'fabric.conf'),
		INSTANCE_NAME = 'projectname', # Recommend no underscore characters
		REPO = 'http://some.repo.com/projectname/',
	)

my_site()

#--- Set up individual hosts if set_hosts() is inconvenient
def example():
	env.hosts = ['127.0.0.1',]
	update_env()

<commit_msg>Removed Rackspace code from example.<commit_after>"""
fab deployment script
====================================

"""
import os

from fabric.api import *
import fabric.colors

from fab_deploy import *

def my_site():
	""" Default Configuration """
	env.conf = dict(
		PROVIDER = 'ec2_us_east', # Also can use ec2_us_west
		AWS_ACCESS_KEY_ID     = '',
		AWS_SECRET_ACCESS_KEY = '',

		CONF_FILE = os.path.join(os.getcwd(),'fabric.conf'),
		INSTANCE_NAME = 'projectname', # Recommend no underscore characters
		REPO = 'http://some.repo.com/projectname/',
	)

my_site()

#--- Set up individual hosts if set_hosts() is inconvenient
def example():
	env.hosts = ['127.0.0.1',]
	update_env()

<|endoftext|>


<commit_before>from app import app, db
from flask.ext.login import login_required, current_user
from flask import request, redirect, url_for, jsonify
from models import Post
from urllib.parse import urljoin
import views

import requests
import json

@app.route('/admin/authorize_facebook')
@login_required
def authorize_facebook():
    import urllib.parse
    import urllib.request
    redirect_uri = app.config.get('SITE_URL') + '/admin/authorize_facebook'
    params = {'client_id': app.config.get('FACEBOOK_APP_ID'),
              'redirect_uri': redirect_uri,
              'scope': 'publish_stream'}

    code = request.args.get('code')
    if code:
        params['code'] = code
        params['client_secret'] = app.config.get('FACEBOOK_APP_SECRET')

        r = urllib.request.urlopen(
            'https://graph.facebook.com/oauth/access_token?'
            + urllib.parse.urlencode(params))
        payload = urllib.parse.parse_qs(r.read())
        access_token = payload[b'access_token']
        current_user.facebook_access_token = access_token
        db.session.commit()
        return redirect(url_for('settings'))
    else:
        return redirect('https://graph.facebook.com/oauth/authorize?'
                        + urllib.parse.urlencode(params))

@app.route('/api/syndicate_to_facebook', methods=['POST'])
@login_required
def syndicate_to_facebook():
    try:
        post_id = int(request.form.get('post_id'))
        post = Post.query.filter_by(id=post_id).first()
        handle_new_or_edit(post)
        db.session.commit()
        return jsonify(success=True, facebook_post_id=post.facebook_post_id,
                       facebook_permalink=post.facebook_url)
    except Exception as e:
        app.logger.exception('posting to facebook')
        response = jsonify(success=False,
                           error="exception while syndicating to Facebook: {}"
                           .format(e))
        return response

def handle_new_or_edit(post):
    app.logger.debug('publishing to facebook')

    share_link = next((share_context.source for share_context
                       in post.share_contexts), None)

    actions = {'name': 'See Original',
               'link': post.permalink}
    privacy = {'value': 'EVERYONE'}

    img_url = views.get_first_image(post.content, post.content_format)
    if img_url:
        img_url = urljoin(app.config['SITE_URL'], img_url)

    post_args = {'access_token': post.author.facebook_access_token,
                 'name': post.title,
                 'message': views.format_as_text(post.content,
                                                 post.content_format),
                 'link': share_link,
                 'picture': img_url,
                 'actions': json.dumps(actions),
                 'privact': json.dumps(privacy)}

    if post.facebook_post_id:
        response = requests.post('http://graph.facebook.com/{}'
                                 .format(post.facebook_post_id),
                                 data=post_args)
    else:
        response = requests.post('https://graph.facebook.com/me/feed',
                                 data=post_args)

    app.logger.debug("Got response from facebook %s", response)

    if response.status_code // 100 != 2:
        raise RuntimeError("Bad response from Facebook. Status: {}, Content: {}"
                           .format(response.status_code, response.content))

    if 'json' in response.headers['content-type']:
        result = response.json()

    app.logger.debug('published to facebook. response {}'.format(result))
    if result:
        post.facebook_post_id = result['id']
<commit_msg>rawr typo<commit_after>from app import app, db
from flask.ext.login import login_required, current_user
from flask import request, redirect, url_for, jsonify
from models import Post
from urllib.parse import urljoin
import views

import requests
import json

@app.route('/admin/authorize_facebook')
@login_required
def authorize_facebook():
    import urllib.parse
    import urllib.request
    redirect_uri = app.config.get('SITE_URL') + '/admin/authorize_facebook'
    params = {'client_id': app.config.get('FACEBOOK_APP_ID'),
              'redirect_uri': redirect_uri,
              'scope': 'publish_stream'}

    code = request.args.get('code')
    if code:
        params['code'] = code
        params['client_secret'] = app.config.get('FACEBOOK_APP_SECRET')

        r = urllib.request.urlopen(
            'https://graph.facebook.com/oauth/access_token?'
            + urllib.parse.urlencode(params))
        payload = urllib.parse.parse_qs(r.read())
        access_token = payload[b'access_token']
        current_user.facebook_access_token = access_token
        db.session.commit()
        return redirect(url_for('settings'))
    else:
        return redirect('https://graph.facebook.com/oauth/authorize?'
                        + urllib.parse.urlencode(params))

@app.route('/api/syndicate_to_facebook', methods=['POST'])
@login_required
def syndicate_to_facebook():
    try:
        post_id = int(request.form.get('post_id'))
        post = Post.query.filter_by(id=post_id).first()
        handle_new_or_edit(post)
        db.session.commit()
        return jsonify(success=True, facebook_post_id=post.facebook_post_id,
                       facebook_permalink=post.facebook_url)
    except Exception as e:
        app.logger.exception('posting to facebook')
        response = jsonify(success=False,
                           error="exception while syndicating to Facebook: {}"
                           .format(e))
        return response

def handle_new_or_edit(post):
    app.logger.debug('publishing to facebook')

    share_link = next((share_context.source for share_context
                       in post.share_contexts), None)

    actions = {'name': 'See Original',
               'link': post.permalink}
    privacy = {'value': 'EVERYONE'}

    img_url = views.get_first_image(post.content, post.content_format)
    if img_url:
        img_url = urljoin(app.config['SITE_URL'], img_url)

    post_args = {'access_token': post.author.facebook_access_token,
                 'name': post.title,
                 'message': views.format_as_text(post.content,
                                                 post.content_format),
                 'link': share_link,
                 'picture': img_url,
                 'actions': json.dumps(actions),
                 'privact': json.dumps(privacy)}

    if post.facebook_post_id:
        response = requests.post('https://graph.facebook.com/{}'
                                 .format(post.facebook_post_id),
                                 data=post_args)
    else:
        response = requests.post('https://graph.facebook.com/me/feed',
                                 data=post_args)

    app.logger.debug("Got response from facebook %s", response)

    if response.status_code // 100 != 2:
        raise RuntimeError("Bad response from Facebook. Status: {}, Content: {}"
                           .format(response.status_code, response.content))

    if 'json' in response.headers['content-type']:
        result = response.json()

    app.logger.debug('published to facebook. response {}'.format(result))
    if result:
        post.facebook_post_id = result['id']
<|endoftext|>


<commit_before># coding=utf-8

from __future__ import unicode_literals

import re
import random

_re_token = re.compile(r'\{\{(\s?)(\w+)(\s?)\}\}')

class Generator(object):

    __config = {}

    def __init__(self, **config):
        self.providers = []
        self.__config = dict(
            list(self.__config.items()) + list(config.items()))

    def add_provider(self, provider):

        if type(provider) is type:
            provider = provider(self)

        self.providers.insert(0, provider)

        for method_name in dir(provider):
            # skip 'private' method
            if method_name.startswith('_'):
                continue

            faker_function = getattr(provider, method_name)

            if hasattr(faker_function, '__call__') or \
                    isinstance(faker_function, (classmethod, staticmethod)):
                # add all faker method to generator
                self.set_formatter(method_name, faker_function)

    def provider(self, name):
        try:
            return list(filter(lambda p: p.__provider__ == name.lower(),
                               self.get_providers()))[0]
        except IndexError:
            return None

    def get_providers(self):
        """Returns added providers."""
        return self.providers

    def seed(self, seed=None):
        """Calls random.seed"""
        random.seed(seed)

    def format(self, formatter, *args, **kwargs):
        """
        This is a secure way to make a fake from another Provider.
        """
        # TODO: data export?
        return self.get_formatter(formatter)(*args, **kwargs)

    def get_formatter(self, formatter):
        try:
            return getattr(self, formatter)
        except AttributeError:
            raise AttributeError('Unknown formatter "{0}"'.format(formatter))

    def set_formatter(self, name, method):
        """
        This method adds a provider method to generator.
        Override this method to add some decoration or logging stuff.
        """
        setattr(self, name, method)

    def parse(self, text):
        """
        Replaces tokens (like '{{ tokenName }}' or '{{tokenName}}')
        with the result from the token method call.
        """
        return _re_token.sub(self.__format_token, text)

    def __format_token(self, matches):
        formatter = list(matches.groups())
        formatter[1] = self.format(formatter[1])
        return "".join(formatter)
<commit_msg>Prefer singe-quoted string literals<commit_after># coding=utf-8

from __future__ import unicode_literals

import re
import random

_re_token = re.compile(r'\{\{(\s?)(\w+)(\s?)\}\}')

class Generator(object):

    __config = {}

    def __init__(self, **config):
        self.providers = []
        self.__config = dict(
            list(self.__config.items()) + list(config.items()))

    def add_provider(self, provider):

        if type(provider) is type:
            provider = provider(self)

        self.providers.insert(0, provider)

        for method_name in dir(provider):
            # skip 'private' method
            if method_name.startswith('_'):
                continue

            faker_function = getattr(provider, method_name)

            if hasattr(faker_function, '__call__') or \
                    isinstance(faker_function, (classmethod, staticmethod)):
                # add all faker method to generator
                self.set_formatter(method_name, faker_function)

    def provider(self, name):
        try:
            return list(filter(lambda p: p.__provider__ == name.lower(),
                               self.get_providers()))[0]
        except IndexError:
            return None

    def get_providers(self):
        """Returns added providers."""
        return self.providers

    def seed(self, seed=None):
        """Calls random.seed"""
        random.seed(seed)

    def format(self, formatter, *args, **kwargs):
        """
        This is a secure way to make a fake from another Provider.
        """
        # TODO: data export?
        return self.get_formatter(formatter)(*args, **kwargs)

    def get_formatter(self, formatter):
        try:
            return getattr(self, formatter)
        except AttributeError:
            raise AttributeError('Unknown formatter "{0}"'.format(formatter))

    def set_formatter(self, name, method):
        """
        This method adds a provider method to generator.
        Override this method to add some decoration or logging stuff.
        """
        setattr(self, name, method)

    def parse(self, text):
        """
        Replaces tokens (like '{{ tokenName }}' or '{{tokenName}}')
        with the result from the token method call.
        """
        return _re_token.sub(self.__format_token, text)

    def __format_token(self, matches):
        formatter = list(matches.groups())
        formatter[1] = self.format(formatter[1])
        return ''.join(formatter)
<|endoftext|>


<commit_before>from volume import mkfs as make_volume
from volume import find_metadata_path
from volume import FarmFSVolume
from fs import normalize
from fs import entries

def mkfs(args):
  make_volume(args.root)
  print "FileSystem Created %s" % args.root
  exit(0)

def writekey(args):
  vol = FarmFSVolume(find_metadata_path(normalize('.')))
  db = vol.keydb
  value = db.write(args.key, args.value)
  exit(0)

def readkey(args):
  vol = FarmFSVolume(find_metadata_path(normalize('.')))
  db = vol.keydb
  value = db.read(args.key)
  if value is not None:
    print value
  exit(0)

def list_keys(args):
  vol = FarmFSVolume(find_metadata_path(normalize('.')))
  db = vol.keydb
  for key in db.list():
    print key

def findvol(args):
  root = find_metadata_path(normalize('.'))
  print "Volume found at: %s" % root
  exit(0)

def freeze(args):
  vol = FarmFSVolume(find_metadata_path(normalize('.')))
  vol.freeze(map(normalize, args.files))

def thaw(args):
  vol = FarmFSVolume(find_metadata_path(normalize('.')))
  vol.thaw(map(normalize, args.files))

def fsck(args):
  retcode = 0
  vol = FarmFSVolume(find_metadata_path(normalize('.')))
  for bad_hash in vol.check_userdata_hashes():
    print "CORRUPTION: checksum mismatch in ", bad_hash
    retcode = 1
  for bad_link in vol.check_inbound_links():
    print "CORRUPTION: broken link in ", bad_link
    retcode = 1
  if retcode == 0:
    print "fsck found no issues"
  exit(retcode)

def walk(args):
  vol = FarmFSVolume(find_metadata_path(normalize('.')))
  if args.walk == "root":
    parents = map(normalize, vol.roots())
    exclude = vol.mdd
    match = ["file", "dir", "link"]
  elif args.walk == "userdata":
    parents = map(normalize, [vol.udd])
    exclude = vol.mdd
    match = ["file"]
  elif args.walk == "keys":
    parents = map(normalize, [vol.keydbd])
    exclude = vol.mdd
    match = ["file"]
  else:
    raise ValueException("Unknown walk: %s" % args.walk)
  walk = entries(parents, exclude)
  for path, type_ in walk:
    if type_ in match:
      print type_, path

def count(args):
  vol = FarmFSVolume(find_metadata_path(normalize('.')))
  counts = vol.count()
  for f, c in counts.items():
    print c, f

def reverse(args):
  vol = FarmFSVolume(find_metadata_path(normalize('.')))
  for x in vol.reverse(args.udd_name):
    print x

def status(args):
  vol = FarmFSVolume(find_metadata_path(normalize('.')))
  paths = map(normalize, args.paths)
  for thawed in vol.thawed(paths):
    print thawed

def gc(args):
  vol = FarmFSVolume(find_metadata_path(normalize('.')))
  for f in vol.gc():
    print "Removing", f

def snap(args):
  vol = FarmFSVolume(find_metadata_path(normalize('.')))
  snapdb = vol.snapdb
  if args.action == 'make':
    vol.snap(args.name)
  elif args.action == 'list':
    for snap in snapdb.list():
      print snap
  elif args.action == 'read':
    snap = snapdb.get(args.name)
    for (type_, path, ud_path) in snap:
      print type_, path, ud_path
  elif args.action == 'delete':
    snapdb.delete(args.name)
  elif args.action == 'restore':
    raise NotImplemented("restore is not a snap verb yet")
  else:
    raise ValueError("Unknown action %s in snap command" % args.action)

<commit_msg>fix snap read verb. badly.<commit_after>from volume import mkfs as make_volume
from volume import find_metadata_path
from volume import FarmFSVolume
from fs import normalize
from fs import entries

def mkfs(args):
  make_volume(args.root)
  print "FileSystem Created %s" % args.root
  exit(0)

def writekey(args):
  vol = FarmFSVolume(find_metadata_path(normalize('.')))
  db = vol.keydb
  value = db.write(args.key, args.value)
  exit(0)

def readkey(args):
  vol = FarmFSVolume(find_metadata_path(normalize('.')))
  db = vol.keydb
  value = db.read(args.key)
  if value is not None:
    print value
  exit(0)

def list_keys(args):
  vol = FarmFSVolume(find_metadata_path(normalize('.')))
  db = vol.keydb
  for key in db.list():
    print key

def findvol(args):
  root = find_metadata_path(normalize('.'))
  print "Volume found at: %s" % root
  exit(0)

def freeze(args):
  vol = FarmFSVolume(find_metadata_path(normalize('.')))
  vol.freeze(map(normalize, args.files))

def thaw(args):
  vol = FarmFSVolume(find_metadata_path(normalize('.')))
  vol.thaw(map(normalize, args.files))

def fsck(args):
  retcode = 0
  vol = FarmFSVolume(find_metadata_path(normalize('.')))
  for bad_hash in vol.check_userdata_hashes():
    print "CORRUPTION: checksum mismatch in ", bad_hash
    retcode = 1
  for bad_link in vol.check_inbound_links():
    print "CORRUPTION: broken link in ", bad_link
    retcode = 1
  if retcode == 0:
    print "fsck found no issues"
  exit(retcode)

def walk(args):
  vol = FarmFSVolume(find_metadata_path(normalize('.')))
  if args.walk == "root":
    parents = map(normalize, vol.roots())
    exclude = vol.mdd
    match = ["file", "dir", "link"]
  elif args.walk == "userdata":
    parents = map(normalize, [vol.udd])
    exclude = vol.mdd
    match = ["file"]
  elif args.walk == "keys":
    parents = map(normalize, [vol.keydbd])
    exclude = vol.mdd
    match = ["file"]
  else:
    raise ValueException("Unknown walk: %s" % args.walk)
  walk = entries(parents, exclude)
  for path, type_ in walk:
    if type_ in match:
      print type_, path

def count(args):
  vol = FarmFSVolume(find_metadata_path(normalize('.')))
  counts = vol.count()
  for f, c in counts.items():
    print c, f

def reverse(args):
  vol = FarmFSVolume(find_metadata_path(normalize('.')))
  for x in vol.reverse(args.udd_name):
    print x

def status(args):
  vol = FarmFSVolume(find_metadata_path(normalize('.')))
  paths = map(normalize, args.paths)
  for thawed in vol.thawed(paths):
    print thawed

def gc(args):
  vol = FarmFSVolume(find_metadata_path(normalize('.')))
  for f in vol.gc():
    print "Removing", f

def snap(args):
  vol = FarmFSVolume(find_metadata_path(normalize('.')))
  snapdb = vol.snapdb
  if args.action == 'make':
    vol.snap(args.name)
  elif args.action == 'list':
    for snap in snapdb.list():
      print snap
  elif args.action == 'read':
    snap = snapdb.get(args.name)
    for i in snap:
      print i._type, i._ref, i._path #TODO IMPL DETAILS ARE HERE!
  elif args.action == 'delete':
    snapdb.delete(args.name)
  elif args.action == 'restore':
    raise NotImplemented("restore is not a snap verb yet")
  else:
    raise ValueError("Unknown action %s in snap command" % args.action)

<|endoftext|>


<commit_before>
from django.conf import settings
from django.core.urlresolvers import reverse
from django.http import HttpResponseRedirect, Http404, HttpResponseForbidden
from django.shortcuts import render_to_response, render, get_object_or_404
from django.template import RequestContext
from django.views import static
from django.views.decorators.http import require_POST

from wiki.forms import RevisionForm
from wiki.hooks import hookset
from wiki.models import Page, Revision, MediaFile

from base.decorators import profile_required

@profile_required
def page(request, slug, binder, *args, **kwargs):
    wiki = binder.lookup(*args, **kwargs)
    try:
        if wiki:
            page = wiki.pages.get(slug=slug)
        else:
            page = Page.objects.get(slug=slug)
        if not hookset.can_view_page(page, request.user):
            raise Http404()

        try:
            rev = page.revisions.get(pk=request.GET.get("rev", "-1"))
        except Revision.DoesNotExist:
            rev = page.revisions.latest()
    except Page.DoesNotExist:
        return HttpResponseRedirect(binder.edit_url(wiki, slug))

    return render_to_response("wiki/page.html", {
        "revision": rev,
        "can_edit": hookset.can_edit_page(page, request.user),
    }, context_instance=RequestContext(request))

@profile_required
def edit(request, slug, binder, *args, **kwargs):
    wiki = binder.lookup(*args, **kwargs)
    try:
        if wiki:
            page = wiki.pages.get(slug=slug)
        else:
            page = Page.objects.get(slug=slug)
        rev = page.revisions.latest()
        if not hookset.can_edit_page(page, request.user):
            return HttpResponseForbidden()
    except Page.DoesNotExist:
        return HttpResponseRedirect(reverse("wiki_add") + "?slug=" + slug)

    form = RevisionForm(
        request.POST if "edit" in request.POST else None,
        revision=rev,
    )
    if form.is_valid():
        revision = form.save(commit=False)
        revision.page = page
        revision.created_by = request.user
        revision.created_ip = request.META.get(settings.WIKI_IP_ADDRESS_META_FIELD, "REMOTE_ADDR")
        revision.parse()
        revision.save()
        return HttpResponseRedirect(binder.page_url(wiki, slug))

    form.fields["content"].help_text = ""

    can_delete = hookset.can_delete_page(page, request.user) and page.pk

    if can_delete and "delete" in request.POST:
        page.delete()
        return HttpResponseRedirect(reverse("wiki_all"))

    page_name = "Edit {0}".format(page.slug)

    return render_to_response("wiki/edit.html", {
        "page_name": page_name,
        "form": form,
        "page": page,
        "can_delete": can_delete,
    }, context_instance=RequestContext(request))

@profile_required
def add_page_view(request, binder, *args, **kwargs):
    wiki = binder.lookup(*args, **kwargs)
    slug = request.GET.get("slug", "Page Name")

    try:
        if wiki:
            page = wiki.pages.get(slug=slug)
        else:
            page = Page.objects.get(slug=slug)
    except Page.DoesNotExist:
        pass
    else:
        return HttpResponseRedirect(page.get_edit_url())

    if not hookset.can_create_page(wiki, request.user):
        raise Http404()

    form = RevisionForm(
        request.POST if "edit" in request.POST else None,
        revision=None,
    )
    if form.is_valid():
        page = Page.objects.create(wiki=wiki, slug=slug)
        revision = form.save(commit=False)
        revision.page = page
        revision.created_by = request.user
        revision.created_ip = request.META.get(settings.WIKI_IP_ADDRESS_META_FIELD, "REMOTE_ADDR")
        revision.parse()
        revision.save()
        return HttpResponseRedirect(binder.page_url(wiki, slug))

    form.fields["content"].help_text = ""

    page_name = "Add {0}".format(slug)

    return render_to_response("wiki/edit.html", {
        "page_name": page_name,
        "form": form,
        "can_delete": False,
    }, context_instance=RequestContext(request))

@profile_required
def history_view(request, slug, binder, *args, **kwargs):
    wiki = binder.lookup(*args, **kwargs)
    try:
        if wiki:
            page = wiki.pages.get(slug=slug)
        else:
            page = Page.objects.get(slug=slug)
        rev = page.revisions.latest()
        if not hookset.can_view_page(page, request.user):
            raise Http404()
    except Page.DoesNotExist:
        raise Http404()

    page_name = "History for {0}".format(page.slug)
    return render_to_response("wiki/history.html", {
        "page_name": page_name,
        "page": page,
    }, context_instance=RequestContext(request))

@profile_required
def all_pages_view(request, binder, *args, **kwargs):
    wiki = binder.lookup(*args, **kwargs)
    if wiki:
        pages = wiki.pages.all()
    else:
        pages = Page.objects.all()

    pages = [
        page
        for page in pages
        if hookset.can_view_page(page, request.user)
    ]

    page_name = "All Pages"
    can_add = hookset.can_create_page(wiki, request.user)
    return render_to_response("wiki/all.html", {
        "page_name": page_name,
        "pages": pages,
        "can_add": can_add,
    }, context_instance=RequestContext(request))
<commit_msg>Fixed default slug name<commit_after>
from django.conf import settings
from django.core.urlresolvers import reverse
from django.http import HttpResponseRedirect, Http404, HttpResponseForbidden
from django.shortcuts import render_to_response, render, get_object_or_404
from django.template import RequestContext
from django.views import static
from django.views.decorators.http import require_POST

from wiki.forms import RevisionForm
from wiki.hooks import hookset
from wiki.models import Page, Revision, MediaFile

from base.decorators import profile_required

@profile_required
def page(request, slug, binder, *args, **kwargs):
    wiki = binder.lookup(*args, **kwargs)
    try:
        if wiki:
            page = wiki.pages.get(slug=slug)
        else:
            page = Page.objects.get(slug=slug)
        if not hookset.can_view_page(page, request.user):
            raise Http404()

        try:
            rev = page.revisions.get(pk=request.GET.get("rev", "-1"))
        except Revision.DoesNotExist:
            rev = page.revisions.latest()
    except Page.DoesNotExist:
        return HttpResponseRedirect(binder.edit_url(wiki, slug))

    return render_to_response("wiki/page.html", {
        "revision": rev,
        "can_edit": hookset.can_edit_page(page, request.user),
    }, context_instance=RequestContext(request))

@profile_required
def edit(request, slug, binder, *args, **kwargs):
    wiki = binder.lookup(*args, **kwargs)
    try:
        if wiki:
            page = wiki.pages.get(slug=slug)
        else:
            page = Page.objects.get(slug=slug)
        rev = page.revisions.latest()
        if not hookset.can_edit_page(page, request.user):
            return HttpResponseForbidden()
    except Page.DoesNotExist:
        return HttpResponseRedirect(reverse("wiki_add") + "?slug=" + slug)

    form = RevisionForm(
        request.POST if "edit" in request.POST else None,
        revision=rev,
    )
    if form.is_valid():
        revision = form.save(commit=False)
        revision.page = page
        revision.created_by = request.user
        revision.created_ip = request.META.get(settings.WIKI_IP_ADDRESS_META_FIELD, "REMOTE_ADDR")
        revision.parse()
        revision.save()
        return HttpResponseRedirect(binder.page_url(wiki, slug))

    form.fields["content"].help_text = ""

    can_delete = hookset.can_delete_page(page, request.user) and page.pk

    if can_delete and "delete" in request.POST:
        page.delete()
        return HttpResponseRedirect(reverse("wiki_all"))

    page_name = "Edit {0}".format(page.slug)

    return render_to_response("wiki/edit.html", {
        "page_name": page_name,
        "form": form,
        "page": page,
        "can_delete": can_delete,
    }, context_instance=RequestContext(request))

@profile_required
def add_page_view(request, binder, *args, **kwargs):
    wiki = binder.lookup(*args, **kwargs)
    slug = request.GET.get("slug", "")
    if not slug:
        slug = "Page Name"

    try:
        if wiki:
            page = wiki.pages.get(slug=slug)
        else:
            page = Page.objects.get(slug=slug)
    except Page.DoesNotExist:
        pass
    else:
        return HttpResponseRedirect(page.get_edit_url())

    if not hookset.can_create_page(wiki, request.user):
        raise Http404()

    form = RevisionForm(
        request.POST if "edit" in request.POST else None,
        revision=None,
    )
    if form.is_valid():
        page = Page.objects.create(wiki=wiki, slug=slug)
        revision = form.save(commit=False)
        revision.page = page
        revision.created_by = request.user
        revision.created_ip = request.META.get(settings.WIKI_IP_ADDRESS_META_FIELD, "REMOTE_ADDR")
        revision.parse()
        revision.save()
        return HttpResponseRedirect(binder.page_url(wiki, slug))

    form.fields["content"].help_text = ""

    page_name = "Add {0}".format(slug)

    return render_to_response("wiki/edit.html", {
        "page_name": page_name,
        "form": form,
        "can_delete": False,
    }, context_instance=RequestContext(request))

@profile_required
def history_view(request, slug, binder, *args, **kwargs):
    wiki = binder.lookup(*args, **kwargs)
    try:
        if wiki:
            page = wiki.pages.get(slug=slug)
        else:
            page = Page.objects.get(slug=slug)
        rev = page.revisions.latest()
        if not hookset.can_view_page(page, request.user):
            raise Http404()
    except Page.DoesNotExist:
        raise Http404()

    page_name = "History for {0}".format(page.slug)
    return render_to_response("wiki/history.html", {
        "page_name": page_name,
        "page": page,
    }, context_instance=RequestContext(request))

@profile_required
def all_pages_view(request, binder, *args, **kwargs):
    wiki = binder.lookup(*args, **kwargs)
    if wiki:
        pages = wiki.pages.all()
    else:
        pages = Page.objects.all()

    pages = [
        page
        for page in pages
        if hookset.can_view_page(page, request.user)
    ]

    page_name = "All Pages"
    can_add = hookset.can_create_page(wiki, request.user)
    return render_to_response("wiki/all.html", {
        "page_name": page_name,
        "pages": pages,
        "can_add": can_add,
    }, context_instance=RequestContext(request))
<|endoftext|>


<commit_before>#!/usr/bin/env python3

from bottle import route
import bottle
import tempfile
import os
import shutil
import subprocess

def is_known_format(fmt):
    if fmt == 'std': return True
    if fmt == 'export': return True
    if fmt == 'simple': return True
    if fmt == 'inline': return True
    return False

def get_out_filename(name, fmt = 'std'):
    if not is_known_format(fmt): raise Exception('Unknown format')
    f =  os.path.join(outdir, get_out_filename_rel(name, fmt))
    if not os.path.abspath(f).startswith(os.path.abspath(outdir)):
        raise Exception('Path out problem', os.path.abspath(f), os.path.abspath(outdir))
    return f

def get_out_filename_rel(name, fmt = 'std'):
    if not is_known_format(fmt): raise Exception('Unknown format')
    return "{}.{}.html".format(name, fmt)

def get_in_filename(name, hasExtesion=False):
    f = os.path.join(indir, get_in_filename_rel(name, hasExtesion))
    if not os.path.abspath(f).startswith(os.path.abspath(indir)):
        raise Exception('Path in problem', os.path.abspath(f), os.path.abspath(indir))
    return f

def get_in_filename_rel(name, hasExtesion):
    if not hasExtesion:
        return "{}.md".format(name)
    return name

@route('/view/<name:path><end:re:(.md)?>')
def route_view(name, end):
    md_file = get_in_filename(name)
    if end != "" or os.path.exists(md_file):
        fmt = bottle.request.query.fmt or "std"
        if not is_known_format(fmt): return 'Unknown format'

        compile_md(name, fmt)
        return bottle.static_file(get_out_filename_rel(name, fmt), outdir)

    else:
        return bottle.static_file(get_in_filename_rel(name, True), indir)

@route('/refresh/<name:path>')
def route_refresh(name):
    if needs_update(name):
        return "True"
    return "False"

@route('/')
def route_index():
    import glob

    text = ""

    text += ""
    text += ''.format(style_basic + style_index)
    text += ""
    text += '
Panserver
'
    text += 'Serving Markdown documents rendered using pandoc. By Marcel Fischer'

    def dir_entry(dirname):
        #collect markdown files recursively into a list
        #return '' if no markdown file is in the directory
        dirtext = ''
        for path in sorted(glob.iglob(os.path.join(dirname, '*.md'))):
            if os.path.isdir(path): continue
            d = {}
            d['path'] = path[:-3]
            d['name'] = os.path.basename(path)
            dirtext += '
{name}
'.format(**d)

        for path in sorted(glob.iglob(os.path.join(dirname, '*'))):
            if not os.path.isdir(path): continue
            subdirtext = dir_entry(path)
            #ignore directories with empty listings
            if subdirtext != '':
                dirtext += '
' + os.path.basename(path) + subdirtext + '
'

        if dirtext != '':
            dirtext = '
' + dirtext + '
'
        else:
            dirtext = ' (no documents)'
        return dirtext

    text += "
Directory contents
"
    text += dir_entry('')

    text += ''

    return text

def needs_update(name, fmt = 'std'):
    out_filename = get_out_filename(name, fmt)
    in_filename = get_in_filename(name)
    if not os.path.exists(in_filename): return False
    if not os.path.exists(out_filename): return True
    return os.path.getmtime(out_filename) < os.path.getmtime(in_filename)

def compile_md(name, fmt):
    out_filename = get_out_filename(name, fmt)
    in_filename = get_in_filename(name)
    if needs_update(name, fmt):
        #file is not cached, recompile
        #create directory if needed
        out_filename_dir = os.path.dirname(out_filename)
        if not os.path.exists(out_filename_dir):
            os.makedirs(out_filename_dir)

        #combine action based on format
        action = ['pandoc']

        if fmt == 'std':
            action += ['-H', headerfile, '-B', beforefile,]

        if fmt == 'export':
            action += ['-H', headerfile_export,]

        if fmt != 'inline':
            action += ['-s']

        action += ['-m']

        #main parameters
        action += [in_filename, '-o', out_filename]

        #compile
        print(action)
        subprocess.call(action)

def create_header(autorefresh):
    headertext = ""

    headertext += """{}""".format(meta_no_mobilescale, style_basic + style_document_add)

    if autorefresh:
        headertext += """

"""

    with open(headerfile, 'w') as f:
        f.write(headertext)

def create_header_export():
    text = """{}""".format(meta_no_mobilescale, style_basic)

    with open(headerfile_export, 'w') as f:
        f.write(text)

def create_beforefile():
    beforetext = ""
    beforetext = """
    Panserver: Index
    Format:
    Export
    Simple
    Inline

    """
    with open(beforefile, 'w') as f:
        f.write(beforetext)

#global vars
tempdir = tempfile.mkdtemp()
outdir = os.path.join(tempdir, "out")
indir = os.path.abspath('.')

headerfile = os.path.join(tempdir, "header.html")
headerfile_export = os.path.join(tempdir, "header.export.html")
beforefile = os.path.join(tempdir, "before.html")

if not os.path.exists(outdir):
    os.makedirs(outdir)

meta_no_mobilescale = """<meta name="viewport" content="width=device-width, initial-scale=1.0">"""

style_basic = """
        body {
            width: 92%; max-width: 45em;  margin: auto;
            font-size: 1.2rem; line-height: 1.5}
        code { font-family: "DejaVu Sans Mono", Consolas, monospace; }
        .LaTeX { font-size: 1.2em; }
        div.figure .caption {  padding: 0em 1em 1em; font-size: 0.8em; }
        ul li { list-style-type: disc; }
        li p { margin: 0.5em 0; }
"""

style_document_add = """
        .topmenu { font-size: 1em; color: lightgrey; }
        .topmenu a { color: lightgrey; }
"""
style_index = """
        ul.file-listing a { text-decoration: none; }
        ul.file-listing ul.file-listing { padding-left: 1.5em; }
        ul li.dir-entry {
            list-style-image: url('data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABQAAAAUCAYAAACNiR0NAAAApElEQVQ4je3UsQqBYRTG8Z9SMpjsrsAdYMI1WNyDhdnKriiD1U3YXML3SWE3uAgDX+lF8iqlPPX01qnn/55OncMvqYMUSeA1KjHAFKUH9S7GMcDkSb2Jk/vOQ2/QhzKq2F/fd52/ftzCBKZYYRnhA2ohcIb6m+PJdJv9A//AT4GND4BZtp0BBxgiFwG8zY7QgyIWLsv96gCE3uKIHeYoRDT1ZZ0BsR1JIXdX8A8AAAAASUVORK5CYII=');
        }
        ul li.file-entry{
            list-style-image: url('data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABQAAAAUCAYAAACNiR0NAAAA00lEQVQ4jcXUQUqCURSG4Qc0bKaUiKsIhEbazMBhOHILES3EUYIGtpBwlDODhhFCTqQdtIYGf5H+eOVeFP4P3ul7L4fzHbZziQme9jDEqYhU8IkervbwjecYaRPziIeXuIuRNvESKRQjjRV+4f6XN8xQPkQ4wO0Ga9QPEeazKER4gi6uA3T9zy1KeCZb8McAY1RThCkpRtjAh2yJd/EuG0txPzy68BxT4RM2RS1FWEZH+IS1UUoRpuQ1JPw7sDfCDcnTx0rWrJ1pYSTckDwPuNgU/ADJ7ku1B60bXwAAAABJRU5ErkJggg==');
        }
"""

if __name__ == '__main__':
    import webbrowser
    import argparse

    parser = argparse.ArgumentParser(description='Run a local markdown compiling server')

    parser.add_argument('-a', action='store_const', const=True)
    parser.add_argument('-p', '--port', type=int, default=8080)
    parser.add_argument('-b', action='store_const', const=True)
    parser.add_argument('-r', action='store_const', const=True)
    parser.add_argument('path', nargs='?')
    config = parser.parse_args()

    create_header(config.a)
    create_header_export()
    create_beforefile()

    if config.path != None:
        if os.path.isdir(config.path):
            os.chdir(config.path)
            indir = os.path.abspath('.')
        else:
            raise Exception('Unknown path argument')

    if config.b:
        webbrowser.get().open('http://localhost:{}/'.format(config.port))

    host = 'localhost'
    if config.r:
        host = ''

    bottle.run(host=host, port=config.port)
    shutil.rmtree(outdir)

<commit_msg>Fix pre overflow<commit_after>#!/usr/bin/env python3

from bottle import route
import bottle
import tempfile
import os
import shutil
import subprocess

def is_known_format(fmt):
    if fmt == 'std': return True
    if fmt == 'export': return True
    if fmt == 'simple': return True
    if fmt == 'inline': return True
    return False

def get_out_filename(name, fmt = 'std'):
    if not is_known_format(fmt): raise Exception('Unknown format')
    f =  os.path.join(outdir, get_out_filename_rel(name, fmt))
    if not os.path.abspath(f).startswith(os.path.abspath(outdir)):
        raise Exception('Path out problem', os.path.abspath(f), os.path.abspath(outdir))
    return f

def get_out_filename_rel(name, fmt = 'std'):
    if not is_known_format(fmt): raise Exception('Unknown format')
    return "{}.{}.html".format(name, fmt)

def get_in_filename(name, hasExtesion=False):
    f = os.path.join(indir, get_in_filename_rel(name, hasExtesion))
    if not os.path.abspath(f).startswith(os.path.abspath(indir)):
        raise Exception('Path in problem', os.path.abspath(f), os.path.abspath(indir))
    return f

def get_in_filename_rel(name, hasExtesion):
    if not hasExtesion:
        return "{}.md".format(name)
    return name

@route('/view/<name:path><end:re:(.md)?>')
def route_view(name, end):
    md_file = get_in_filename(name)
    if end != "" or os.path.exists(md_file):
        fmt = bottle.request.query.fmt or "std"
        if not is_known_format(fmt): return 'Unknown format'

        compile_md(name, fmt)
        return bottle.static_file(get_out_filename_rel(name, fmt), outdir)

    else:
        return bottle.static_file(get_in_filename_rel(name, True), indir)

@route('/refresh/<name:path>')
def route_refresh(name):
    if needs_update(name):
        return "True"
    return "False"

@route('/')
def route_index():
    import glob

    text = ""

    text += ""
    text += ''.format(style_basic + style_index)
    text += ""
    text += '
Panserver
'
    text += 'Serving Markdown documents rendered using pandoc. By Marcel Fischer'

    def dir_entry(dirname):
        #collect markdown files recursively into a list
        #return '' if no markdown file is in the directory
        dirtext = ''
        for path in sorted(glob.iglob(os.path.join(dirname, '*.md'))):
            if os.path.isdir(path): continue
            d = {}
            d['path'] = path[:-3]
            d['name'] = os.path.basename(path)
            dirtext += '
{name}
'.format(**d)

        for path in sorted(glob.iglob(os.path.join(dirname, '*'))):
            if not os.path.isdir(path): continue
            subdirtext = dir_entry(path)
            #ignore directories with empty listings
            if subdirtext != '':
                dirtext += '
' + os.path.basename(path) + subdirtext + '
'

        if dirtext != '':
            dirtext = '
' + dirtext + '
'
        else:
            dirtext = ' (no documents)'
        return dirtext

    text += "
Directory contents
"
    text += dir_entry('')

    text += ''

    return text

def needs_update(name, fmt = 'std'):
    out_filename = get_out_filename(name, fmt)
    in_filename = get_in_filename(name)
    if not os.path.exists(in_filename): return False
    if not os.path.exists(out_filename): return True
    return os.path.getmtime(out_filename) < os.path.getmtime(in_filename)

def compile_md(name, fmt):
    out_filename = get_out_filename(name, fmt)
    in_filename = get_in_filename(name)
    if needs_update(name, fmt):
        #file is not cached, recompile
        #create directory if needed
        out_filename_dir = os.path.dirname(out_filename)
        if not os.path.exists(out_filename_dir):
            os.makedirs(out_filename_dir)

        #combine action based on format
        action = ['pandoc']

        if fmt == 'std':
            action += ['-H', headerfile, '-B', beforefile,]

        if fmt == 'export':
            action += ['-H', headerfile_export,]

        if fmt != 'inline':
            action += ['-s']

        action += ['-m']

        #main parameters
        action += [in_filename, '-o', out_filename]

        #compile
        print(action)
        subprocess.call(action)

def create_header(autorefresh):
    headertext = ""

    headertext += """{}""".format(meta_no_mobilescale, style_basic + style_document_add)

    if autorefresh:
        headertext += """

"""

    with open(headerfile, 'w') as f:
        f.write(headertext)

def create_header_export():
    text = """{}""".format(meta_no_mobilescale, style_basic)

    with open(headerfile_export, 'w') as f:
        f.write(text)

def create_beforefile():
    beforetext = ""
    beforetext = """
    Panserver: Index
    Format:
    Export
    Simple
    Inline

    """
    with open(beforefile, 'w') as f:
        f.write(beforetext)

#global vars
tempdir = tempfile.mkdtemp()
outdir = os.path.join(tempdir, "out")
indir = os.path.abspath('.')

headerfile = os.path.join(tempdir, "header.html")
headerfile_export = os.path.join(tempdir, "header.export.html")
beforefile = os.path.join(tempdir, "before.html")

if not os.path.exists(outdir):
    os.makedirs(outdir)

meta_no_mobilescale = """<meta name="viewport" content="width=device-width, initial-scale=1.0">"""

style_basic = """
        body {
            width: 92%; max-width: 45em;  margin: auto;
            font-size: 1.2rem; line-height: 1.5}
        code { font-family: "DejaVu Sans Mono", Consolas, monospace; }
        pre { overflow: auto; }
        .LaTeX { font-size: 1.2em; }
        div.figure .caption {  padding: 0em 1em 1em; font-size: 0.8em; }
        ul li { list-style-type: disc; }
        li p { margin: 0.5em 0; }
"""

style_document_add = """
        .topmenu { font-size: 1em; color: lightgrey; }
        .topmenu a { color: lightgrey; }
"""
style_index = """
        ul.file-listing a { text-decoration: none; }
        ul.file-listing ul.file-listing { padding-left: 1.5em; }
        ul li.dir-entry {
            list-style-image: url('data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABQAAAAUCAYAAACNiR0NAAAApElEQVQ4je3UsQqBYRTG8Z9SMpjsrsAdYMI1WNyDhdnKriiD1U3YXML3SWE3uAgDX+lF8iqlPPX01qnn/55OncMvqYMUSeA1KjHAFKUH9S7GMcDkSb2Jk/vOQ2/QhzKq2F/fd52/ftzCBKZYYRnhA2ohcIb6m+PJdJv9A//AT4GND4BZtp0BBxgiFwG8zY7QgyIWLsv96gCE3uKIHeYoRDT1ZZ0BsR1JIXdX8A8AAAAASUVORK5CYII=');
        }
        ul li.file-entry{
            list-style-image: url('data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABQAAAAUCAYAAACNiR0NAAAA00lEQVQ4jcXUQUqCURSG4Qc0bKaUiKsIhEbazMBhOHILES3EUYIGtpBwlDODhhFCTqQdtIYGf5H+eOVeFP4P3ul7L4fzHbZziQme9jDEqYhU8IkervbwjecYaRPziIeXuIuRNvESKRQjjRV+4f6XN8xQPkQ4wO0Ga9QPEeazKER4gi6uA3T9zy1KeCZb8McAY1RThCkpRtjAh2yJd/EuG0txPzy68BxT4RM2RS1FWEZH+IS1UUoRpuQ1JPw7sDfCDcnTx0rWrJ1pYSTckDwPuNgU/ADJ7ku1B60bXwAAAABJRU5ErkJggg==');
        }
"""

if __name__ == '__main__':
    import webbrowser
    import argparse

    parser = argparse.ArgumentParser(description='Run a local markdown compiling server')

    parser.add_argument('-a', action='store_const', const=True)
    parser.add_argument('-p', '--port', type=int, default=8080)
    parser.add_argument('-b', action='store_const', const=True)
    parser.add_argument('-r', action='store_const', const=True)
    parser.add_argument('path', nargs='?')
    config = parser.parse_args()

    create_header(config.a)
    create_header_export()
    create_beforefile()

    if config.path != None:
        if os.path.isdir(config.path):
            os.chdir(config.path)
            indir = os.path.abspath('.')
        else:
            raise Exception('Unknown path argument')

    if config.b:
        webbrowser.get().open('http://localhost:{}/'.format(config.port))

    host = 'localhost'
    if config.r:
        host = ''

    bottle.run(host=host, port=config.port)
    shutil.rmtree(outdir)

<|endoftext|>


<commit_before>import networkx
from toolkit import IDLabel
import matplotlib.pyplot as plt
import StringIO
from networkx import *

############### AUX CLASSES ###################
class Word(IDLabel):
    def __init__(self, id, word, lemma):
        IDLabel.__init__(self, id, word)
        self.lemma = lemma

class Lemma(IDLabel):
    def __init__(self, id, lemma, pos):
        IDLabel.__init__(self, id, lemma)
        self.pos = pos

class Relation(IDLabel):
    def __init__(self, id, name):
        IDLabel.__init__(self, id, name)

_words, _lemmata = {}, {}
def getWord(wid, word, lemmaid, lemma, pos):
    if wid not in _words:
        if lemmaid not in _lemmata:
            _lemmata[lemmaid] = Lemma(lemmaid, lemma, pos)
        _words[wid] = Word(wid, word, _lemmata[lemmaid])
    return _words[wid]

_rels = None
def getRels(db):
    global _rels
    if _rels is None:
        _rels = {}
        for relid, name in db.doQuery("select relid, name from parses_rels"):
            _rels[relid] = name
    return _rels

############# PARSE TREE #####################

class ParseNode(IDLabel):
    def __init__(self, tree, word, position, data={}):
        IDLabel.__init__(self, position, word.label)
        self.tree = tree
        self.word = word
        self.position = position
        self.data = {}
    def getChildren(self):
        for dummy, child in self.tree.graph.out_edges(self):
            yield child
    def getRelations(self):
        for dummy, child in self.tree.graph.out_edges(self):
            d = self.tree.graph.get_edge_data(self, child)
            yield child, d.get("rel")
    def addData(self, key, value):
        self.data[key] = value
    def getRelation(self, other):
        for dummy, child in self.tree.graph.out_edges(self):
            if child == other:
                d = self.tree.graph.get_edge_data(self, child)
                return d.get('rel')
    def identity(self):
        return self.tree, self.position
    def __str__(self):
        return "%i:%s" % (self.position, self.word)
    def __repr__(self):
        return "ParseNode(%r, %r)" % (self.word, self.position)

class ParseTree(IDLabel):
    def __init__(self, id):
        IDLabel.__init__(self, id, str(id))
        self.graph = networkx.DiGraph()
    def addTriple(self, parent, child, rel):
        self.graph.add_edge(parent, child, rel=rel)

    def getNodes(self):
        """ Returns the tree nodes from root, breadth first """
        stack = [self.getRoot()]
        seen = set(stack)
        while stack:
            n = stack.pop(0)
            yield n
            for n2 in n.getChildren():
                if n2 in seen: continue
                stack.append(n2)
                seen.add(n2)

    def getNode(self, position):
        for node in self.graph.nodes():
            if node.position == position:
                return node

    def getRoot(self):
        for node in self.graph.nodes():
            if not self.graph.in_edges(node):
                return node
        for node in self.graph.nodes():
            if node.position == 0:
                return node
        raise Exception("No root! Sentence id %i " % self.id)

    def printTree(self, **kargs):
        kargs2 = dict(labelattr='label', edgelabelattr='getRelation')
        kargs2.update(kargs)
        return printGraph(self.graph,  **kargs2)

def fromDB(db, sid):
    SQL = "select wordbegin, wordid, word, lemmaid, lemma, pos from vw_parses_words where sentenceid = %i" % sid
    words = {}
    p = ParseTree(sid)
    for data in db.doQuery(SQL):
        i, wdata = data[0], data[1:]
        words[i] = ParseNode(p, getWord(*wdata), i)
    rels = getRels(db)
    SQL = "select parentbegin, childbegin, relation from parses_triples where sentenceid = %i order by childbegin" % sid
    for pb, cb, rel in db.doQuery(SQL):
        p.addTriple(words[pb], words[cb], rels[rel])
    return p

#################### Printing #######################

def getAttr(obj, attr):
    try:
        return obj.__getattribute__(attr)
    except AttributeError:
        return None

def printGraph(graph, output=None, labelattr="nx_label", edgelabelattr="nx_edgelabeler", nodeattr="nx_node_attributes"):
    plt.clf()
    pos=graphviz_layout(graph,prog='dot')
    draw_networkx_edges(graph, pos, width=1,alpha=0.2,edge_color='b')

    labels = {}
    edgelabels = {}
    for n in graph.nodes():
        label = getAttr(n, labelattr)
        if label: labels[n] = label
        edgelabeler = getAttr(n, edgelabelattr)
        if edgelabeler:
            for n2 in graph.nodes():
                el = edgelabeler(n2)
                if el: edgelabels[n, n2] = el
        nodeattrs = getAttr(n, nodeattr)
        if nodeattrs:
            draw_networkx_nodes(graph, pos, nodelist=[n], **nodeattrs)

    draw_networkx_labels(graph,pos,labels,font_size=9)
    draw_networkx_edge_labels(graph, pos, edge_labels=edgelabels, font_size=7)

    plt.axis('off')
    if output:
        return plt.savefig(output)
    else:
        s = StringIO.StringIO()
        plt.savefig(s, format="png")
        return s.getvalue()

############## DRIVER ######################

if __name__ == '__main__':
    import dbtoolkit
    db = dbtoolkit.amcatDB()
    t = fromDB(db, 192)
    t.getRoot().nx_node_attributes = dict(color="r")
    t.printTree(output="/home/amcat/www-plain/test/test.png")

<commit_msg>[svn r1743] parsepicture<commit_after>import networkx
from toolkit import IDLabel
import matplotlib.pyplot as plt
import StringIO
from networkx import *

############### AUX CLASSES ###################
class Word(IDLabel):
    def __init__(self, id, word, lemma):
        IDLabel.__init__(self, id, word)
        self.lemma = lemma

class Lemma(IDLabel):
    def __init__(self, id, lemma, pos):
        IDLabel.__init__(self, id, lemma)
        self.pos = pos

class Relation(IDLabel):
    def __init__(self, id, name):
        IDLabel.__init__(self, id, name)

_words, _lemmata = {}, {}
def getWord(wid, word, lemmaid, lemma, pos):
    if wid not in _words:
        if lemmaid not in _lemmata:
            _lemmata[lemmaid] = Lemma(lemmaid, lemma, pos)
        _words[wid] = Word(wid, word, _lemmata[lemmaid])
    return _words[wid]

_rels = None
def getRels(db):
    global _rels
    if _rels is None:
        _rels = {}
        for relid, name in db.doQuery("select relid, name from parses_rels"):
            _rels[relid] = name
    return _rels

############# PARSE TREE #####################

class ParseNode(IDLabel):
    def __init__(self, tree, word, position, data={}):
        IDLabel.__init__(self, position, word.label)
        self.tree = tree
        self.word = word
        self.position = position
        self.data = {}
    def getChildren(self):
        for dummy, child in self.tree.graph.out_edges(self):
            yield child
    def getRelations(self):
        for dummy, child in self.tree.graph.out_edges(self):
            d = self.tree.graph.get_edge_data(self, child)
            yield child, d.get("rel")
    def addData(self, key, value):
        self.data[key] = value
    def getRelation(self, other):
        for dummy, child in self.tree.graph.out_edges(self):
            if child == other:
                d = self.tree.graph.get_edge_data(self, child)
                return d.get('rel')
    def identity(self):
        return self.tree, self.position
    def __str__(self):
        return "%i:%s" % (self.position, self.word)
    def __repr__(self):
        return "ParseNode(%r, %r)" % (self.word, self.position)

class ParseTree(IDLabel):
    def __init__(self, id):
        IDLabel.__init__(self, id, str(id))
        self.graph = networkx.DiGraph()
    def addTriple(self, parent, child, rel):
        self.graph.add_edge(parent, child, rel=rel)

    def getNodes(self):
        """ Returns the tree nodes from root, breadth first """
        stack = [self.getRoot()]
        seen = set(stack)
        while stack:
            n = stack.pop(0)
            yield n
            for n2 in n.getChildren():
                if n2 in seen: continue
                stack.append(n2)
                seen.add(n2)

    def getNode(self, position):
        for node in self.graph.nodes():
            if node.position == position:
                return node

    def getRoot(self):
        for node in self.graph.nodes():
            if not self.graph.in_edges(node):
                return node
        for node in self.graph.nodes():
            if node.position == 0:
                return node
        raise Exception("No root! Sentence id %i " % self.id)

    def printTree(self, **kargs):
        kargs2 = dict(labelattr='label', edgelabelattr='getRelation')
        kargs2.update(kargs)
        return printGraph(self.graph,  **kargs2)

def fromDB(db, sid):
    SQL = "select wordbegin, wordid, word, lemmaid, lemma, pos from vw_parses_words where sentenceid = %i" % sid
    words = {}
    p = ParseTree(sid)
    rels = db.doQuery(SQL)
    if not rels: return
    for data in rels:
        i, wdata = data[0], data[1:]
        words[i] = ParseNode(p, getWord(*wdata), i)
    rels = getRels(db)
    SQL = "select parentbegin, childbegin, relation from parses_triples where sentenceid = %i order by childbegin" % sid
    for pb, cb, rel in db.doQuery(SQL):
        p.addTriple(words[pb], words[cb], rels[rel])
    return p

#################### Printing #######################

def getAttr(obj, attr):
    try:
        return obj.__getattribute__(attr)
    except AttributeError:
        return None

def printGraph(graph, output=None, labelattr="nx_label", edgelabelattr="nx_edgelabeler", nodeattr="nx_node_attributes"):
    plt.clf()
    pos=graphviz_layout(graph,prog='dot')
    draw_networkx_edges(graph, pos, width=1,alpha=0.2,edge_color='b')

    labels = {}
    edgelabels = {}
    for n in graph.nodes():
        label = getAttr(n, labelattr)
        if label: labels[n] = label
        edgelabeler = getAttr(n, edgelabelattr)
        if edgelabeler:
            for n2 in graph.nodes():
                el = edgelabeler(n2)
                if el: edgelabels[n, n2] = el
        nodeattrs = getAttr(n, nodeattr)
        if nodeattrs:
            draw_networkx_nodes(graph, pos, nodelist=[n], **nodeattrs)

    draw_networkx_labels(graph,pos,labels,font_size=9)
    draw_networkx_edge_labels(graph, pos, edge_labels=edgelabels, font_size=7)

    plt.axis('off')
    if output:
        return plt.savefig(output)
    else:
        s = StringIO.StringIO()
        plt.savefig(s, format="png")
        return s.getvalue()

def getSentencePicture(db, sid):
    t = fromDB(db, sid)
    if not t: return
    return t.printTree()

############## DRIVER ######################

if __name__ == '__main__':
    import dbtoolkit
    db = dbtoolkit.amcatDB()
    t = fromDB(db, 192)
    t.getRoot().nx_node_attributes = dict(color="r")
    t.printTree(output="/home/amcat/www-plain/test/test.png")

<|endoftext|>


<commit_before>import math

def rgb_clamp(vals):
    return tuple(_adjusted_round(max(0, min(255, c))) for c in vals)

def _adjusted_round(n):
    if math.modf(n)[0] == 0.5:
        return int(math.ceil(n))
    else:
        return int(round(n))
<commit_msg>add docstring to converters.rgb_clamp()<commit_after>import math

def rgb_clamp(vals):
    """Clamp the value of each RGB component to the range of 0 to 255"""
    return tuple(_adjusted_round(max(0, min(255, c))) for c in vals)

def _adjusted_round(n):
    if math.modf(n)[0] == 0.5:
        return int(math.ceil(n))
    else:
        return int(round(n))
<|endoftext|>