<commit_before>#!/usr/bin/env python
"""

This file is part of the mumax-ec2 package.

Copyright (c) 2014-2015 Colin Jermain, Graham Rowlands

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.

"""

__version__ = 1.0

__TAG__ = u'mumax-ec2'
__STATUS__ = u'status'
__READY__ = u'1'
PORT = 35367
__ON_AWS__ = "==================== AWS INSTANCE ===================="

import argparse
import boto.ec2
import paramiko
import sys, os
from time import sleep
import select
from sshtunnel import SSHTunnelForwarder

# Start the config parser
import ConfigParser
config = ConfigParser.ConfigParser()
config.read(os.path.dirname(os.path.realpath(__file__))+"/config.ini")

# Connect to Amazon Web Services (AWS)
conn = boto.ec2.connect_to_region(config.get('EC2', 'Region'),
            aws_access_key_id=config.get('EC2', 'AccessID'),
            aws_secret_access_key=config.get('EC2', 'SecretKey'))

up_condition = lambda i: (
    i.state == u'running'
)

mumax_ec2_condition = lambda i: (
    __TAG__ in i.tags
)

ready_condition = lambda i: (
    __STATUS__ in i.tags and
    i.tags[__STATUS__] == __READY__
)

instances = conn.get_only_instances()
mumax_ec2_instances = [i for i in instances if mumax_ec2_condition(i)]
ready_instances = [i for i in instances if (
    mumax_ec2_condition(i) and ready_condition(i) and up_condition(i)
)]

def has_id(id, running=True):
    """ Returns True if the ID is a valid mumax-ec2 instance
    and if it is running or not
    """
    for instance in mumax_ec2_instances:
        if instance.id == id:
            if up_condition(instance) == running:
                return True
            else:
                return False
    return False

def launch_instance():
    """ Launch a new AWS instance """
    print "Creating a new instance of %s" % config.get('EC2', 'Image')
    reservation = conn.run_instances(
        config.get('EC2', 'Image'),
        key_name=config.get('EC2', 'PrivateKeyName'),
        instance_type=config.get('EC2', 'InstanceType'),
        security_groups=config.get('EC2', 'SecurityGroups').split(',')
    )
    instance = reservation.instances[0]
    instance.add_tag(__TAG__, __version__)
    instance.add_tag(__STATUS__, __READY__)
    return instance

def wait_for_instance(instance, delay=10):
    """ Waits for an instance to boot up
    """
    print "Waiting for instance %s to boot up..." % instance.id
    while instance.state != u'running':
        sleep(delay)
        instance.update()
    sleep(delay)
    print "Instance %s is ready" % instance.id

def get_ready_instance():
    """ Returns an instance from the ready list or launches
    a new instance upon prompt
    """
    if len(ready_instances) == 0:
        print "There are no instances waiting to be used."
        answer = raw_input("Create a new instance for this job? [Yn]: ")
        if len(answer) == 0 or answer.startswith(("Y", "y")):
            instance = launch_instance()
            wait_for_instance(instance)
            return instance
        else:
            print "No instance will be launched"
            return None
    else:
        instance = ready_instances[0] # Select the 1st ready instance
        print "Instance %s is ready" % instance.id
        return instance

def run(args):
    """Run the mumax input file on a ready instance
    """
    local_input_file = os.path.realpath(args.filename[0])

    instance = get_ready_instance()
    if instance == None: return None

    try:
        print "Making secure connection to instance %s..." % instance.id
        ssh = paramiko.SSHClient()
        ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
        ssh.connect(instance.public_dns_name,
            username=config.get('EC2', 'User'),
            key_filename=config.get('EC2', 'PrivateKeyFile')
        )
        sftp = ssh.open_sftp()
    except:
        print "Could not connect to remote server"
        return

    instance.remove_tag(__STATUS__)
    basename = os.path.basename(local_input_file)
    directory = "/home/%s" % config.get('EC2', 'User')
    input_file = "%s/simulations/%s" % (directory, basename)
    output_dir = "%s/simulations/%s" % (directory, basename.replace(".mx3", ".out"))

    print "Transferring input file to instance: %s" % basename
    sftp.put(local_input_file, input_file)

    print "Starting port forwarding"
    mykey = paramiko.RSAKey.from_private_key_file(config.get('EC2', 'PrivateKeyFile'))
    server = SSHTunnelForwarder(
        ssh_address=(instance.public_dns_name, 22),
        ssh_username=config.get('EC2', 'User'),
        ssh_private_key=mykey,
        remote_bind_address=('127.0.0.1', PORT),
        local_bind_address=('127.0.0.1', PORT)
    )
    server.start()

    transport = ssh.get_transport()
    channel = transport.open_session()

    binary = "%s/mumax3/%s" % (directory, config.get('EC2', 'MuMaxBinary'))
    cmd = "source ./include_cuda && %s -f -http=:%i %s" % (binary, PORT, input_file)
    print cmd
    print __ON_AWS__
    channel.exec_command(cmd)

    #TODO: Test blocking ability
    try:
        while not channel.exit_status_ready():
            if channel.recv_ready():
                print channel.recv(1024), # ending comma to prevent newline
        # Finish reading everything
        while channel.recv_ready():
            print channel.recv(1024), # ending comma to prevent newline

    except KeyboardInterrupt:
        print "\nCaught Ctrl-C"
        channel.close()

        try:
            # Try to kill the process
            ssh.get_transport().open_session().exec_command(
                "kill -9 `ps -fu | grep mumax3 | grep -v grep | awk '{print $2}'`"
            )
        except:
            pass

    print __ON_AWS__
    print "Receiving output files from instance"
    local_output_dir = local_input_file.replace(".mx3", ".out")
    if not os.path.isdir(local_output_dir):
        os.mkdir(local_output_dir)
    os.chdir(local_output_dir)
    sftp.chdir(output_dir)
    files = sftp.listdir()
    for f in files:
        sftp.get(f, f)

    print "Removing simulation files from instance"
    sftp.remove(input_file)
    ssh.exec_command("rm -r %s" % output_dir)

    print "Stopping port forwarding"
    server.stop()

    ssh.close()

    answer = raw_input("Terminate the instance? [Yn]: ")
    if len(answer) == 0 or answer.startswith(("Y", "y")):
        print "Terminating instance"
        conn.terminate_instances(instance_ids=[instance.id])
    else:
        instance.add_tag(__STATUS__, __READY__)
        answer = raw_input("Stop the instance? [Yn]: ")
        if len(answer) == 0 or answer.startswith(("Y", "y")):
            print "Stopping instance"
            conn.stop_instances(instance_ids=[instance.id])
        else:
            print "The instance has been left running"

def list_instances(args):
    if len(mumax_ec2_instances) > 0:
        print "Mumax-ec2 Instances:"
        print "    ID\t\tIP\t\tStatus"
        for instance in mumax_ec2_instances:
            if mumax_ec2_condition(instance):
                if up_condition(instance):
                    if ready_condition(instance):
                        status = "ready"
                    else:
                        status = "running"
                else:
                    status = "stopped"
                print "    %s\t%s\t(%s)" % (instance.id, instance.ip_address, status)
    else:
        print "No mumax-ec2 instances currently running"

def _launch_instance(args):
    instance = launch_instance()
    if args.wait:
        wait_for_instance(instance)

def terminate_instance(args):
    if has_id(args.id[0], running=True):
        print "Terminating instance %s" % args.id[0]
        conn.terminate_instances(instance_ids=args.id)
    else:
        print "AWS ID %s is not a valid mumax-ec2 instance" % args.id[0]

def stop_instance(args):
    if has_id(args.id[0], running=True):
        print "Stopping instance %s" % args.id[0]
        conn.stop_instances(instance_ids=args.id)
    else:
        print "AWS ID %s is not a valid mumax-ec2 instance" % args.id[0]

def start_instance(args):
    if has_id(args.id[0], running=False):
        print "Starting instance %s" % args.id[0]
        conn.start_instances(instance_ids=args.id)
        if args.wait:
            for instance in mumax_ec2_instances:
                if instance.id == args.id[0]:
                    break
            wait_for_instance(instance)
    else:
        print "AWS ID %s is not a valid mumax-ec2 instance" % args.id[0]

if __name__ == '__main__':
    parser = argparse.ArgumentParser(
        description='Runs MuMax3 .mx3 files on Amazon Web Services (AWS) instances')
    subparsers = parser.add_subparsers(help='sub-command help')

    parser_run = subparsers.add_parser('run', help='run help')
    parser_run.add_argument('filename', metavar='filename', type=str, nargs=1,
        help='A .mx3 input file for MuMax3')
    parser_run.set_defaults(func=run)

    parser_list = subparsers.add_parser('list', help='list help')
    parser_list.set_defaults(func=list_instances)

    parser_launch = subparsers.add_parser('launch', help='launch help')
    parser_launch.add_argument('--wait', action='store_true')
    parser_launch.set_defaults(func=_launch_instance, wait=False)

    parser_terminate = subparsers.add_parser('terminate', help='terminate help')
    parser_terminate.add_argument('id', metavar='aws_id', type=str, nargs=1,
        help='AWS ID of instance')
    parser_terminate.set_defaults(func=terminate_instance)

    parser_stop = subparsers.add_parser('stop', help='stop help')
    parser_stop.add_argument('id', metavar='aws_id', type=str, nargs=1,
        help='AWS ID of instance')
    parser_stop.set_defaults(func=stop_instance)

    parser_start = subparsers.add_parser('start', help='start help')
    parser_start.add_argument('id', metavar='aws_id', type=str, nargs=1,
        help='AWS ID of instance')
    parser_start.add_argument('--wait', action='store_true')
    parser_start.set_defaults(func=start_instance)

    args = parser.parse_args()
    args.func(args)
<commit_msg>Moved port forward stop earlier to prevent errors<commit_after>#!/usr/bin/env python
"""

This file is part of the mumax-ec2 package.

Copyright (c) 2014-2015 Colin Jermain, Graham Rowlands

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.

"""

__version__ = 1.0

__TAG__ = u'mumax-ec2'
__STATUS__ = u'status'
__READY__ = u'1'
PORT = 35367
__ON_AWS__ = "==================== AWS INSTANCE ===================="

import argparse
import boto.ec2
import paramiko
import sys, os
from time import sleep
import select
from sshtunnel import SSHTunnelForwarder

# Start the config parser
import ConfigParser
config = ConfigParser.ConfigParser()
config.read(os.path.dirname(os.path.realpath(__file__))+"/config.ini")

# Connect to Amazon Web Services (AWS)
conn = boto.ec2.connect_to_region(config.get('EC2', 'Region'),
            aws_access_key_id=config.get('EC2', 'AccessID'),
            aws_secret_access_key=config.get('EC2', 'SecretKey'))

up_condition = lambda i: (
    i.state == u'running'
)

mumax_ec2_condition = lambda i: (
    __TAG__ in i.tags
)

ready_condition = lambda i: (
    __STATUS__ in i.tags and
    i.tags[__STATUS__] == __READY__
)

instances = conn.get_only_instances()
mumax_ec2_instances = [i for i in instances if mumax_ec2_condition(i)]
ready_instances = [i for i in instances if (
    mumax_ec2_condition(i) and ready_condition(i) and up_condition(i)
)]

def has_id(id, running=True):
    """ Returns True if the ID is a valid mumax-ec2 instance
    and if it is running or not
    """
    for instance in mumax_ec2_instances:
        if instance.id == id:
            if up_condition(instance) == running:
                return True
            else:
                return False
    return False

def launch_instance():
    """ Launch a new AWS instance """
    print "Creating a new instance of %s" % config.get('EC2', 'Image')
    reservation = conn.run_instances(
        config.get('EC2', 'Image'),
        key_name=config.get('EC2', 'PrivateKeyName'),
        instance_type=config.get('EC2', 'InstanceType'),
        security_groups=config.get('EC2', 'SecurityGroups').split(',')
    )
    instance = reservation.instances[0]
    instance.add_tag(__TAG__, __version__)
    instance.add_tag(__STATUS__, __READY__)
    return instance

def wait_for_instance(instance, delay=10):
    """ Waits for an instance to boot up
    """
    print "Waiting for instance %s to boot up..." % instance.id
    while instance.state != u'running':
        sleep(delay)
        instance.update()
    sleep(delay)
    print "Instance %s is ready" % instance.id

def get_ready_instance():
    """ Returns an instance from the ready list or launches
    a new instance upon prompt
    """
    if len(ready_instances) == 0:
        print "There are no instances waiting to be used."
        answer = raw_input("Create a new instance for this job? [Yn]: ")
        if len(answer) == 0 or answer.startswith(("Y", "y")):
            instance = launch_instance()
            wait_for_instance(instance)
            return instance
        else:
            print "No instance will be launched"
            return None
    else:
        instance = ready_instances[0] # Select the 1st ready instance
        print "Instance %s is ready" % instance.id
        return instance

def run(args):
    """Run the mumax input file on a ready instance
    """
    local_input_file = os.path.realpath(args.filename[0])

    instance = get_ready_instance()
    if instance == None: return None

    try:
        print "Making secure connection to instance %s..." % instance.id
        ssh = paramiko.SSHClient()
        ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
        ssh.connect(instance.public_dns_name,
            username=config.get('EC2', 'User'),
            key_filename=config.get('EC2', 'PrivateKeyFile')
        )
        sftp = ssh.open_sftp()
    except:
        print "Could not connect to remote server"
        return

    instance.remove_tag(__STATUS__)
    basename = os.path.basename(local_input_file)
    directory = "/home/%s" % config.get('EC2', 'User')
    input_file = "%s/simulations/%s" % (directory, basename)
    output_dir = "%s/simulations/%s" % (directory, basename.replace(".mx3", ".out"))

    print "Transferring input file to instance: %s" % basename
    sftp.put(local_input_file, input_file)

    print "Starting port forwarding"
    mykey = paramiko.RSAKey.from_private_key_file(config.get('EC2', 'PrivateKeyFile'))
    server = SSHTunnelForwarder(
        ssh_address=(instance.public_dns_name, 22),
        ssh_username=config.get('EC2', 'User'),
        ssh_private_key=mykey,
        remote_bind_address=('127.0.0.1', PORT),
        local_bind_address=('127.0.0.1', PORT)
    )
    server.start()

    transport = ssh.get_transport()
    channel = transport.open_session()

    binary = "%s/mumax3/%s" % (directory, config.get('EC2', 'MuMaxBinary'))
    cmd = "source ./include_cuda && %s -f -http=:%i %s" % (binary, PORT, input_file)
    print cmd
    print __ON_AWS__
    channel.exec_command(cmd)

    #TODO: Test blocking ability
    try:
        while not channel.exit_status_ready():
            if channel.recv_ready():
                print channel.recv(1024), # ending comma to prevent newline
        # Finish reading everything
        while channel.recv_ready():
            print channel.recv(1024), # ending comma to prevent newline

    except KeyboardInterrupt:
        print "\nCaught Ctrl-C"
        channel.close()

        try:
            # Try to kill the process
            ssh.get_transport().open_session().exec_command(
                "kill -9 `ps -fu | grep mumax3 | grep -v grep | awk '{print $2}'`"
            )
        except:
            pass

    print __ON_AWS__
    print "Stopping port forwarding"
    server.stop()

    print "Receiving output files from instance"
    local_output_dir = local_input_file.replace(".mx3", ".out")
    if not os.path.isdir(local_output_dir):
        os.mkdir(local_output_dir)
    os.chdir(local_output_dir)
    sftp.chdir(output_dir)
    files = sftp.listdir()
    for f in files:
        sftp.get(f, f)

    print "Removing simulation files from instance"
    sftp.remove(input_file)
    ssh.exec_command("rm -r %s" % output_dir)

    ssh.close()

    answer = raw_input("Terminate the instance? [Yn]: ")
    if len(answer) == 0 or answer.startswith(("Y", "y")):
        print "Terminating instance"
        conn.terminate_instances(instance_ids=[instance.id])
    else:
        instance.add_tag(__STATUS__, __READY__)
        answer = raw_input("Stop the instance? [Yn]: ")
        if len(answer) == 0 or answer.startswith(("Y", "y")):
            print "Stopping instance"
            conn.stop_instances(instance_ids=[instance.id])
        else:
            print "The instance has been left running"

def list_instances(args):
    if len(mumax_ec2_instances) > 0:
        print "Mumax-ec2 Instances:"
        print "    ID\t\tIP\t\tStatus"
        for instance in mumax_ec2_instances:
            if mumax_ec2_condition(instance):
                if up_condition(instance):
                    if ready_condition(instance):
                        status = "ready"
                    else:
                        status = "running"
                else:
                    status = "stopped"
                print "    %s\t%s\t(%s)" % (instance.id, instance.ip_address, status)
    else:
        print "No mumax-ec2 instances currently running"

def _launch_instance(args):
    instance = launch_instance()
    if args.wait:
        wait_for_instance(instance)

def terminate_instance(args):
    if has_id(args.id[0], running=True):
        print "Terminating instance %s" % args.id[0]
        conn.terminate_instances(instance_ids=args.id)
    else:
        print "AWS ID %s is not a valid mumax-ec2 instance" % args.id[0]

def stop_instance(args):
    if has_id(args.id[0], running=True):
        print "Stopping instance %s" % args.id[0]
        conn.stop_instances(instance_ids=args.id)
    else:
        print "AWS ID %s is not a valid mumax-ec2 instance" % args.id[0]

def start_instance(args):
    if has_id(args.id[0], running=False):
        print "Starting instance %s" % args.id[0]
        conn.start_instances(instance_ids=args.id)
        if args.wait:
            for instance in mumax_ec2_instances:
                if instance.id == args.id[0]:
                    break
            wait_for_instance(instance)
    else:
        print "AWS ID %s is not a valid mumax-ec2 instance" % args.id[0]

if __name__ == '__main__':
    parser = argparse.ArgumentParser(
        description='Runs MuMax3 .mx3 files on Amazon Web Services (AWS) instances')
    subparsers = parser.add_subparsers(help='sub-command help')

    parser_run = subparsers.add_parser('run', help='run help')
    parser_run.add_argument('filename', metavar='filename', type=str, nargs=1,
        help='A .mx3 input file for MuMax3')
    parser_run.set_defaults(func=run)

    parser_list = subparsers.add_parser('list', help='list help')
    parser_list.set_defaults(func=list_instances)

    parser_launch = subparsers.add_parser('launch', help='launch help')
    parser_launch.add_argument('--wait', action='store_true')
    parser_launch.set_defaults(func=_launch_instance, wait=False)

    parser_terminate = subparsers.add_parser('terminate', help='terminate help')
    parser_terminate.add_argument('id', metavar='aws_id', type=str, nargs=1,
        help='AWS ID of instance')
    parser_terminate.set_defaults(func=terminate_instance)

    parser_stop = subparsers.add_parser('stop', help='stop help')
    parser_stop.add_argument('id', metavar='aws_id', type=str, nargs=1,
        help='AWS ID of instance')
    parser_stop.set_defaults(func=stop_instance)

    parser_start = subparsers.add_parser('start', help='start help')
    parser_start.add_argument('id', metavar='aws_id', type=str, nargs=1,
        help='AWS ID of instance')
    parser_start.add_argument('--wait', action='store_true')
    parser_start.set_defaults(func=start_instance)

    args = parser.parse_args()
    args.func(args)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
#
# KeyRock documentation build configuration file, created by
# sphinx-quickstart on Tue Oct  6 12:58:37 2015.
#
# This file is execfile()d with the current directory set to its
# containing dir.
#
# Note that not all possible configuration values are present in this
# autogenerated file.
#
# All configuration values have a default; values that are commented out
# serve to show the default.

import sys
import os
import shlex

# If extensions (or modules to document with autodoc) are in another directory,
# add these directories to sys.path here. If the directory is relative to the
# documentation root, use os.path.abspath to make it absolute, like shown here.
#sys.path.insert(0, os.path.abspath('.'))

# -- General configuration ------------------------------------------------

# If your documentation needs a minimal Sphinx version, state it here.
#needs_sphinx = '1.0'

# Add any Sphinx extension module names here, as strings. They can be
# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
# ones.
extensions = [
    'sphinx.ext.autodoc',
]

# Add any paths that contain templates here, relative to this directory.
templates_path = ['_templates']

# The suffix(es) of source filenames.
# You can specify multiple suffix as a list of string:
from recommonmark.parser import CommonMarkParser

source_parsers = {
    '.md': CommonMarkParser,
}

source_suffix = ['.rst', '.md']
# source_suffix = '.rst'

# The encoding of source files.
#source_encoding = 'utf-8-sig'

# The master toctree document.
master_doc = 'index'

# General information about the project.
project = u'KeyRock'
copyright = u'2016, Universidad Politecnica de Madrid'
author = u'Universidad Politecnica de Madrid'

# The version info for the project you're documenting, acts as replacement for
# |version| and |release|, also used in various other places throughout the
# built documents.
#
# The short X.Y version.
version = '5.4.0'
# The full version, including alpha/beta/rc tags.
release = '5.4.0'

# The language for content autogenerated by Sphinx. Refer to documentation
# for a list of supported languages.
#
# This is also used if you do content translation via gettext catalogs.
# Usually you set "language" from the command line for these cases.
language = None

# There are two options for replacing |today|: either, you set today to some
# non-false value, then it is used:
#today = ''
# Else, today_fmt is used as the format for a strftime call.
#today_fmt = '%B %d, %Y'

# List of patterns, relative to source directory, that match files and
# directories to ignore when looking for source files.
exclude_patterns = ['_build']

# The reST default role (used for this markup: `text`) to use for all
# documents.
#default_role = None

# If true, '()' will be appended to :func: etc. cross-reference text.
#add_function_parentheses = True

# If true, the current module name will be prepended to all description
# unit titles (such as .. function::).
#add_module_names = True

# If true, sectionauthor and moduleauthor directives will be shown in the
# output. They are ignored by default.
#show_authors = False

# The name of the Pygments (syntax highlighting) style to use.
pygments_style = 'sphinx'

# A list of ignored prefixes for module index sorting.
#modindex_common_prefix = []

# If true, keep warnings as "system message" paragraphs in the built documents.
#keep_warnings = False

# If true, `todo` and `todoList` produce output, else they produce nothing.
todo_include_todos = False

# -- Options for HTML output ----------------------------------------------

# The theme to use for HTML and HTML Help pages.  See the documentation for
# a list of builtin themes.
import os
on_rtd = os.environ.get('READTHEDOCS', None) == 'True'
if on_rtd:
    html_context = {
        'css_files': [
            'https://media.readthedocs.org/css/sphinx_rtd_theme.css',
            'https://media.readthedocs.org/css/readthedocs-doc-embed.css',
            '_static/fiware_style.css',
        ],
    }
else:
    html_theme = 'sphinx_rtd_theme'
    html_context = {
        'css_files': [
            '_static/fiware_style.css',
        ],
    }

# Theme options are theme-specific and customize the look and feel of a theme
# further.  For a list of options available for each theme, see the
# documentation.
#html_theme_options = {}

# Add any paths that contain custom themes here, relative to this directory.
#html_theme_path = []

# The name for this set of Sphinx documents.  If None, it defaults to
# "<project> v<release> documentation".
#html_title = None

# A shorter title for the navigation bar.  Default is the same as html_title.
#html_short_title = None

# The name of an image file (relative to this directory) to place at the top
# of the sidebar.
#html_logo = None

# The name of an image file (within the static path) to use as favicon of the
# docs.  This file should be a Windows icon file (.ico) being 16x16 or 32x32
# pixels large.
#html_favicon = None

# Add any paths that contain custom static files (such as style sheets) here,
# relative to this directory. They are copied after the builtin static files,
# so a file named "default.css" will overwrite the builtin "default.css".
html_static_path = ['_static']

# Add any extra paths that contain custom files (such as robots.txt or
# .htaccess) here, relative to this directory. These files are copied
# directly to the root of the documentation.
#html_extra_path = []

# If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
# using the given strftime format.
#html_last_updated_fmt = '%b %d, %Y'

# If true, SmartyPants will be used to convert quotes and dashes to
# typographically correct entities.
#html_use_smartypants = True

# Custom sidebar templates, maps document names to template names.
#html_sidebars = {}

# Additional templates that should be rendered to pages, maps page names to
# template names.
#html_additional_pages = {}

# If false, no module index is generated.
#html_domain_indices = True

# If false, no index is generated.
#html_use_index = True

# If true, the index is split into individual pages for each letter.
#html_split_index = False

# If true, links to the reST sources are added to the pages.
#html_show_sourcelink = True

# If true, "Created using Sphinx" is shown in the HTML footer. Default is True.
#html_show_sphinx = True

# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True.
#html_show_copyright = True

# If true, an OpenSearch description file will be output, and all pages will
# contain a <link> tag referring to it.  The value of this option must be the
# base URL from which the finished HTML is served.
#html_use_opensearch = ''

# This is the file name suffix for HTML files (e.g. ".xhtml").
#html_file_suffix = None

# Language to be used for generating the HTML full-text search index.
# Sphinx supports the following languages:
#   'da', 'de', 'en', 'es', 'fi', 'fr', 'hu', 'it', 'ja'
#   'nl', 'no', 'pt', 'ro', 'ru', 'sv', 'tr'
#html_search_language = 'en'

# A dictionary with options for the search language support, empty by default.
# Now only 'ja' uses this config value
#html_search_options = {'type': 'default'}

# The name of a javascript file (relative to the configuration directory) that
# implements a search results scorer. If empty, the default will be used.
#html_search_scorer = 'scorer.js'

# Output file base name for HTML help builder.
htmlhelp_basename = 'KeyRockdoc'

# -- Options for LaTeX output ---------------------------------------------

latex_elements = {
# The paper size ('letterpaper' or 'a4paper').
#'papersize': 'letterpaper',

# The font size ('10pt', '11pt' or '12pt').
#'pointsize': '10pt',

# Additional stuff for the LaTeX preamble.
#'preamble': '',

# Latex figure (float) alignment
#'figure_align': 'htbp',
}

# Grouping the document tree into LaTeX files. List of tuples
# (source start file, target name, title,
#  author, documentclass [howto, manual, or own class]).
latex_documents = [
  (master_doc, 'KeyRock.tex', u'KeyRock Documentation',
   u'Enrique Garcia Navalon', 'manual'),
]

# The name of an image file (relative to this directory) to place at the top of
# the title page.
#latex_logo = None

# For "manual" documents, if this is true, then toplevel headings are parts,
# not chapters.
#latex_use_parts = False

# If true, show page references after internal links.
#latex_show_pagerefs = False

# If true, show URL addresses after external links.
#latex_show_urls = False

# Documents to append as an appendix to all manuals.
#latex_appendices = []

# If false, no module index is generated.
#latex_domain_indices = True

# -- Options for manual page output ---------------------------------------

# One entry per manual page. List of tuples
# (source start file, name, description, authors, manual section).
man_pages = [
    (master_doc, 'keyrock', u'KeyRock Documentation',
     [author], 1)
]

# If true, show URL addresses after external links.
#man_show_urls = False

# -- Options for Texinfo output -------------------------------------------

# Grouping the document tree into Texinfo files. List of tuples
# (source start file, target name, title, author,
#  dir menu entry, description, category)
texinfo_documents = [
  (master_doc, 'KeyRock', u'KeyRock Documentation',
   author, 'KeyRock', 'One line description of project.',
   'Miscellaneous'),
]

# Documents to append as an appendix to all manuals.
#texinfo_appendices = []

# If false, no module index is generated.
#texinfo_domain_indices = True

# How to display URL addresses: 'footnote', 'no', or 'inline'.
#texinfo_show_urls = 'footnote'

# If true, do not generate a @detailmenu in the "Top" node's menu.
#texinfo_no_detailmenu = False

# -- Options for Epub output ----------------------------------------------

# Bibliographic Dublin Core info.
epub_title = project
epub_author = author
epub_publisher = author
epub_copyright = copyright

# The basename for the epub file. It defaults to the project name.
#epub_basename = project

# The HTML theme for the epub output. Since the default themes are not optimized
# for small screen space, using the same theme for HTML and epub output is
# usually not wise. This defaults to 'epub', a theme designed to save visual
# space.
#epub_theme = 'epub'

# The language of the text. It defaults to the language option
# or 'en' if the language is not set.
#epub_language = ''

# The scheme of the identifier. Typical schemes are ISBN or URL.
#epub_scheme = ''

# The unique identifier of the text. This can be a ISBN number
# or the project homepage.
#epub_identifier = ''

# A unique identification for the text.
#epub_uid = ''

# A tuple containing the cover image and cover page html template filenames.
#epub_cover = ()

# A sequence of (type, uri, title) tuples for the guide element of content.opf.
#epub_guide = ()

# HTML files that should be inserted before the pages created by sphinx.
# The format is a list of tuples containing the path and title.
#epub_pre_files = []

# HTML files shat should be inserted after the pages created by sphinx.
# The format is a list of tuples containing the path and title.
#epub_post_files = []

# A list of files that should not be packed into the epub file.
epub_exclude_files = ['search.html']

# The depth of the table of contents in toc.ncx.
#epub_tocdepth = 3

# Allow duplicate toc entries.
#epub_tocdup = True

# Choose between 'default' and 'includehidden'.
#epub_tocscope = 'default'

# Fix unsupported image types using the Pillow.
#epub_fix_images = False

# Scale large images.
#epub_max_image_width = 0

# How to display URL addresses: 'footnote', 'no', or 'inline'.
#epub_show_urls = 'inline'

# If false, no index is generated.
#epub_use_index = True
<commit_msg>Updated doc version<commit_after># -*- coding: utf-8 -*-
#
# KeyRock documentation build configuration file, created by
# sphinx-quickstart on Tue Oct  6 12:58:37 2015.
#
# This file is execfile()d with the current directory set to its
# containing dir.
#
# Note that not all possible configuration values are present in this
# autogenerated file.
#
# All configuration values have a default; values that are commented out
# serve to show the default.

import sys
import os
import shlex

# If extensions (or modules to document with autodoc) are in another directory,
# add these directories to sys.path here. If the directory is relative to the
# documentation root, use os.path.abspath to make it absolute, like shown here.
#sys.path.insert(0, os.path.abspath('.'))

# -- General configuration ------------------------------------------------

# If your documentation needs a minimal Sphinx version, state it here.
#needs_sphinx = '1.0'

# Add any Sphinx extension module names here, as strings. They can be
# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
# ones.
extensions = [
    'sphinx.ext.autodoc',
]

# Add any paths that contain templates here, relative to this directory.
templates_path = ['_templates']

# The suffix(es) of source filenames.
# You can specify multiple suffix as a list of string:
from recommonmark.parser import CommonMarkParser

source_parsers = {
    '.md': CommonMarkParser,
}

source_suffix = ['.rst', '.md']
# source_suffix = '.rst'

# The encoding of source files.
#source_encoding = 'utf-8-sig'

# The master toctree document.
master_doc = 'index'

# General information about the project.
project = u'KeyRock'
copyright = u'2016, Universidad Politecnica de Madrid'
author = u'Universidad Politecnica de Madrid'

# The version info for the project you're documenting, acts as replacement for
# |version| and |release|, also used in various other places throughout the
# built documents.
#
# The short X.Y version.
version = '5.4.0'
# The full version, including alpha/beta/rc tags.
release = '5.4.0'

# The language for content autogenerated by Sphinx. Refer to documentation
# for a list of supported languages.
#
# This is also used if you do content translation via gettext catalogs.
# Usually you set "language" from the command line for these cases.
language = None

# There are two options for replacing |today|: either, you set today to some
# non-false value, then it is used:
#today = ''
# Else, today_fmt is used as the format for a strftime call.
#today_fmt = '%B %d, %Y'

# List of patterns, relative to source directory, that match files and
# directories to ignore when looking for source files.
exclude_patterns = ['_build']

# The reST default role (used for this markup: `text`) to use for all
# documents.
#default_role = None

# If true, '()' will be appended to :func: etc. cross-reference text.
#add_function_parentheses = True

# If true, the current module name will be prepended to all description
# unit titles (such as .. function::).
#add_module_names = True

# If true, sectionauthor and moduleauthor directives will be shown in the
# output. They are ignored by default.
#show_authors = False

# The name of the Pygments (syntax highlighting) style to use.
pygments_style = 'sphinx'

# A list of ignored prefixes for module index sorting.
#modindex_common_prefix = []

# If true, keep warnings as "system message" paragraphs in the built documents.
#keep_warnings = False

# If true, `todo` and `todoList` produce output, else they produce nothing.
todo_include_todos = False

# -- Options for HTML output ----------------------------------------------

# The theme to use for HTML and HTML Help pages.  See the documentation for
# a list of builtin themes.
import os
on_rtd = os.environ.get('READTHEDOCS', None) == 'True'
if on_rtd:
    html_context = {
        'css_files': [
            'https://media.readthedocs.org/css/sphinx_rtd_theme.css',
            'https://media.readthedocs.org/css/readthedocs-doc-embed.css',
            '_static/fiware_style.css',
        ],
    }
else:
    html_theme = 'sphinx_rtd_theme'
    html_context = {
        'css_files': [
            '_static/fiware_style.css',
        ],
    }

# Theme options are theme-specific and customize the look and feel of a theme
# further.  For a list of options available for each theme, see the
# documentation.
#html_theme_options = {}

# Add any paths that contain custom themes here, relative to this directory.
#html_theme_path = []

# The name for this set of Sphinx documents.  If None, it defaults to
# "<project> v<release> documentation".
#html_title = None

# A shorter title for the navigation bar.  Default is the same as html_title.
#html_short_title = None

# The name of an image file (relative to this directory) to place at the top
# of the sidebar.
#html_logo = None

# The name of an image file (within the static path) to use as favicon of the
# docs.  This file should be a Windows icon file (.ico) being 16x16 or 32x32
# pixels large.
#html_favicon = None

# Add any paths that contain custom static files (such as style sheets) here,
# relative to this directory. They are copied after the builtin static files,
# so a file named "default.css" will overwrite the builtin "default.css".
html_static_path = ['_static']

# Add any extra paths that contain custom files (such as robots.txt or
# .htaccess) here, relative to this directory. These files are copied
# directly to the root of the documentation.
#html_extra_path = []

# If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
# using the given strftime format.
#html_last_updated_fmt = '%b %d, %Y'

# If true, SmartyPants will be used to convert quotes and dashes to
# typographically correct entities.
#html_use_smartypants = True

# Custom sidebar templates, maps document names to template names.
#html_sidebars = {}

# Additional templates that should be rendered to pages, maps page names to
# template names.
#html_additional_pages = {}

# If false, no module index is generated.
#html_domain_indices = True

# If false, no index is generated.
#html_use_index = True

# If true, the index is split into individual pages for each letter.
#html_split_index = False

# If true, links to the reST sources are added to the pages.
#html_show_sourcelink = True

# If true, "Created using Sphinx" is shown in the HTML footer. Default is True.
#html_show_sphinx = True

# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True.
#html_show_copyright = True

# If true, an OpenSearch description file will be output, and all pages will
# contain a <link> tag referring to it.  The value of this option must be the
# base URL from which the finished HTML is served.
#html_use_opensearch = ''

# This is the file name suffix for HTML files (e.g. ".xhtml").
#html_file_suffix = None

# Language to be used for generating the HTML full-text search index.
# Sphinx supports the following languages:
#   'da', 'de', 'en', 'es', 'fi', 'fr', 'hu', 'it', 'ja'
#   'nl', 'no', 'pt', 'ro', 'ru', 'sv', 'tr'
#html_search_language = 'en'

# A dictionary with options for the search language support, empty by default.
# Now only 'ja' uses this config value
#html_search_options = {'type': 'default'}

# The name of a javascript file (relative to the configuration directory) that
# implements a search results scorer. If empty, the default will be used.
#html_search_scorer = 'scorer.js'

# Output file base name for HTML help builder.
htmlhelp_basename = 'KeyRockdoc'

# -- Options for LaTeX output ---------------------------------------------

latex_elements = {
# The paper size ('letterpaper' or 'a4paper').
#'papersize': 'letterpaper',

# The font size ('10pt', '11pt' or '12pt').
#'pointsize': '10pt',

# Additional stuff for the LaTeX preamble.
#'preamble': '',

# Latex figure (float) alignment
#'figure_align': 'htbp',
}

# Grouping the document tree into LaTeX files. List of tuples
# (source start file, target name, title,
#  author, documentclass [howto, manual, or own class]).
latex_documents = [
  (master_doc, 'KeyRock.tex', u'KeyRock Documentation',
   u'Universidad Politecnica de Madrid', 'manual'),
]

# The name of an image file (relative to this directory) to place at the top of
# the title page.
#latex_logo = None

# For "manual" documents, if this is true, then toplevel headings are parts,
# not chapters.
#latex_use_parts = False

# If true, show page references after internal links.
#latex_show_pagerefs = False

# If true, show URL addresses after external links.
#latex_show_urls = False

# Documents to append as an appendix to all manuals.
#latex_appendices = []

# If false, no module index is generated.
#latex_domain_indices = True

# -- Options for manual page output ---------------------------------------

# One entry per manual page. List of tuples
# (source start file, name, description, authors, manual section).
man_pages = [
    (master_doc, 'keyrock', u'KeyRock Documentation',
     [author], 1)
]

# If true, show URL addresses after external links.
#man_show_urls = False

# -- Options for Texinfo output -------------------------------------------

# Grouping the document tree into Texinfo files. List of tuples
# (source start file, target name, title, author,
#  dir menu entry, description, category)
texinfo_documents = [
  (master_doc, 'KeyRock', u'KeyRock Documentation',
   author, 'KeyRock', 'One line description of project.',
   'Miscellaneous'),
]

# Documents to append as an appendix to all manuals.
#texinfo_appendices = []

# If false, no module index is generated.
#texinfo_domain_indices = True

# How to display URL addresses: 'footnote', 'no', or 'inline'.
#texinfo_show_urls = 'footnote'

# If true, do not generate a @detailmenu in the "Top" node's menu.
#texinfo_no_detailmenu = False

# -- Options for Epub output ----------------------------------------------

# Bibliographic Dublin Core info.
epub_title = project
epub_author = author
epub_publisher = author
epub_copyright = copyright

# The basename for the epub file. It defaults to the project name.
#epub_basename = project

# The HTML theme for the epub output. Since the default themes are not optimized
# for small screen space, using the same theme for HTML and epub output is
# usually not wise. This defaults to 'epub', a theme designed to save visual
# space.
#epub_theme = 'epub'

# The language of the text. It defaults to the language option
# or 'en' if the language is not set.
#epub_language = ''

# The scheme of the identifier. Typical schemes are ISBN or URL.
#epub_scheme = ''

# The unique identifier of the text. This can be a ISBN number
# or the project homepage.
#epub_identifier = ''

# A unique identification for the text.
#epub_uid = ''

# A tuple containing the cover image and cover page html template filenames.
#epub_cover = ()

# A sequence of (type, uri, title) tuples for the guide element of content.opf.
#epub_guide = ()

# HTML files that should be inserted before the pages created by sphinx.
# The format is a list of tuples containing the path and title.
#epub_pre_files = []

# HTML files shat should be inserted after the pages created by sphinx.
# The format is a list of tuples containing the path and title.
#epub_post_files = []

# A list of files that should not be packed into the epub file.
epub_exclude_files = ['search.html']

# The depth of the table of contents in toc.ncx.
#epub_tocdepth = 3

# Allow duplicate toc entries.
#epub_tocdup = True

# Choose between 'default' and 'includehidden'.
#epub_tocscope = 'default'

# Fix unsupported image types using the Pillow.
#epub_fix_images = False

# Scale large images.
#epub_max_image_width = 0

# How to display URL addresses: 'footnote', 'no', or 'inline'.
#epub_show_urls = 'inline'

# If false, no index is generated.
#epub_use_index = True
<|endoftext|>


<commit_before>"""
 Check how much HADSA data we have
"""
import os
import sys
import stat
import psycopg2
IEM = psycopg2.connect(database='iem', host='iemdb', user='nobody')
icursor = IEM.cursor()

def check():
    icursor.execute("""SELECT count(*) from current c JOIN stations s on
    (s.iemid = c.iemid)
    WHERE valid > now() - '75 minutes'::interval and network ~* 'ASOS'""")
    row = icursor.fetchone()

    return row[0]

if __name__ == '__main__':
    count = check()
    if count > 10000:
        print 'OK - %s count |count=%s;1000;5000;10000' % (count, count)
        sys.exit(0)
    elif count > 5000:
        print 'WARNING - %s count |count=%s;1000;5000;10000' % (count, count)
        sys.exit(1)
    else:
        print 'CRITICAL - %s count |count=%s;1000;5000;10000' % (count, count)
        sys.exit(2)<commit_msg>adjust thresholds based on a night's worth of processing<commit_after>"""
 Check how much HADSA data we have
"""
import os
import sys
import stat
import psycopg2
IEM = psycopg2.connect(database='iem', host='iemdb', user='nobody')
icursor = IEM.cursor()

def check():
    icursor.execute("""SELECT count(*) from current c JOIN stations s on
    (s.iemid = c.iemid)
    WHERE valid > now() - '75 minutes'::interval and network ~* 'ASOS'""")
    row = icursor.fetchone()

    return row[0]

if __name__ == '__main__':
    count = check()
    if count > 3000:
        print 'OK - %s count |count=%s;1000;5000;10000' % (count, count)
        sys.exit(0)
    elif count > 2000:
        print 'WARNING - %s count |count=%s;1000;5000;10000' % (count, count)
        sys.exit(1)
    else:
        print 'CRITICAL - %s count |count=%s;1000;5000;10000' % (count, count)
        sys.exit(2)<|endoftext|>


<commit_before>from app.models import Post
from app.utils.helpers import get_or_create
from tests.general import AppTestCase

class TestUtils(AppTestCase):
    def test_get_or_create(self):
        post1, created1 = get_or_create(Post, title='foo', body='bar')
        post1.save()
        post2, created2 = get_or_create(Post, title='foo', body='bar')
        self.assertTrue(created1)
        self.assertFalse(created2)
        self.assertEquals(post1, post2)
<commit_msg>Rename TestUtils test case to TestHelpers<commit_after>from app.models import Post
from app.utils.helpers import get_or_create
from tests.general import AppTestCase

class TestHelpers(AppTestCase):
    def test_get_or_create(self):
        post1, created1 = get_or_create(Post, title='foo', body='bar')
        post1.save()
        post2, created2 = get_or_create(Post, title='foo', body='bar')
        self.assertTrue(created1)
        self.assertFalse(created2)
        self.assertEquals(post1, post2)
<|endoftext|>


<commit_before>import re
from numbers import Number
import urllib

from sqlalchemy.engine import reflection

from flask import Flask, render_template, url_for, abort, request, escape, Markup
from flask.ext.sqlalchemy import SQLAlchemy
from werkzeug import url_encode

import sa_common
import config

app = Flask(__name__)
app.config['SQLALCHEMY_DATABASE_URI'] = config.CONNECTION_STRING
app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False
db = SQLAlchemy(app)

def html_format(html, *args, **kw):
    # escape any values
    args = [escape(arg) for arg in args]
    kw = {k: escape(v) for k, v in kw.items()}
    return Markup(unicode(html).format(*args, **kw))

def add_yes(arg):
    if arg:
        return 'YES'
    return ''

def make_html_span(name, cls):
    return '{{{name}}}'.format(name=name, cls=cls)

def code_desc(arg, code_type):
    codes = codes_data.get(code_type)
    if not codes:
        return arg
    if arg:
        desc = codes.get(arg)
        html = make_html_span('code', 'code') + ' '
        if desc:
            html += make_html_span('desc', 'code_desc')
        else:
            html += make_html_span('desc', 'code_unknown')
            desc = 'UNKNOWN'
        return html_format(html, code=arg, desc=desc)

    html = make_html_span('code', 'code_missing')
    return html_format(html, code='MISSING')

@app.template_global()
def modify_query(**new_values):
    args = request.args.copy()

    for key, value in new_values.items():
        args[key] = value

    return '{}?{}'.format(urllib.quote(request.path), url_encode(args))

@app.template_global()
def cell_function(info, value):
    if len(info) == 1:
        return info[0](value)
    return info[0](value, *info[1:])

@app.template_global()
def make_td_class(arg):
    if isinstance(arg, Number):
        return Markup(' class="numeric"')
    return ''

def run_sql(*args, **kw):
    return sa_common.run_sql(db.engine, *args, **kw)

def get_indexes(*args, **kw):
    return sa_common.get_indexes(db.engine, *args, **kw)

def get_primary_keys(*args, **kw):
    return sa_common.get_primary_keys(db.engine, *args, **kw)

def table_list(*args, **kw):
    return sa_common.table_list(db.engine, *args, **kw)

codes_data = {}
for table in table_list():
    if table.startswith('c_'):
        codes = {}
        sql = 'SELECT code, "desc" FROM "{table}"'.format(table=table)
        result = run_sql(sql)
        for row in result:
            codes[row[0]] = row[1]
        codes_data[table[2:]] = codes

def auto_links(fields):
    links = {}
    for i, field in enumerate(fields):
        if field.get('name') == 'uarn':
            links[i] = ('premises', 'uarn', i)
    return links

def auto_functions(fields):
    functions = {}
    for i, field in enumerate(fields):
        if field.get('name').endswith('_code'):
            functions[i] = (code_desc, field.get('name')[:-5])
    return functions

def show_result(sql, table=None, data=None):
    offset = int(request.args.get('offset', 0))
    # We need to have a result to get the field types
    if data is None:
        data = {}
    result = run_sql(sql + ' LIMIT 1', data)
    fields = sa_common.get_result_fields(db.engine, result, table)
    # Now run query
    result = run_sql(sql + (' LIMIT 1000 OFFSET %s' % offset), data)
    output = {
        'fields': fields,
        'data': result,
        'offset': offset,
    }
    if not 'raw' in request.args:
        output['links'] = auto_links(fields)
        output['functions'] = auto_functions(fields)
    return output

def show_table(table):
    sql = 'SELECT * FROM "%s"' % table
    return show_result(sql, table)

@app.route('/')
def home():
    return render_template('home.html')

@app.route('/table/')
def tables():
    out = []
    for table in sorted(table_list()):
        out.append(table)
    return render_template('tables.html', data=out)

@app.route('/table/
')
def table(table=None):
    if table not in table_list():
        abort(404)
    data = show_table(table)
    data['title'] = table

    return render_template('table_output.html', data=data)

@app.route('/la/')
def la_list():
    sql = '''
    SELECT code, "desc" FROM c_la
    WHERE LEFT(code, 1) IN ('E', 'w')
    ORDER BY "desc"
    '''
    output = show_result(sql)
    output['links'][1] = ('la_premises_list', 'la_code', 0)
    return render_template('table_output.html', data=output)

@app.route('/la/<la_code>')
def la_premises_list(la_code):
    data = {'la_code': la_code}
    sql = '''
    SELECT v.uarn, b.uarn, s.code as scat_code
    FROM vao_list v
    LEFT OUTER JOIN v_vao_base b
    ON b.uarn = v.uarn
    LEFT JOIN c_scat s ON s.code = v.scat_code
    WHERE v.la_code = :la_code
    ORDER BY s.desc
    '''
    output = show_result(sql, data=data)
    del output['links'][1]
    output['functions'][1] = (add_yes,)
    output['fields'][1]['name'] = 'summary'
    return render_template('table_output.html', data=output)

@app.route('/scat/')
def scat_list():
    sql = 'SELECT code, "desc" FROM c_scat ORDER BY "desc"'
    output = show_result(sql)
    output['links'][1] = ('scat_premises_list', 'scat_code', 0)
    return render_template('table_output.html', data=output)

@app.route('/scat/<scat_code>')
def scat_premises_list(scat_code):
    data = {'scat_code': scat_code}
    sql = '''
    SELECT v.uarn, b.uarn, c.code as ba_code
    FROM vao_list v
    LEFT OUTER JOIN v_vao_base b
    ON b.uarn = v.uarn
    LEFT JOIN c_ba c ON c.code = v.ba_code
    WHERE v.scat_code = :scat_code
    ORDER BY c.desc
    '''
    output = show_result(sql, data=data)
    del output['links'][1]
    output['fields'][1]['name'] = 'summary'
    output['functions'][1] = (add_yes,)
    return render_template('table_output.html', data=output)

@app.route('/la_areas/')
def la_areas_list():
    sql = '''
    SELECT code, "desc" FROM c_la
    WHERE LEFT(code, 1) IN ('E', 'w')
    ORDER BY "desc"
    '''
    output = show_result(sql)
    output['links'][1] = ('la_areas', 'la_code', 0)
    return render_template('table_output.html', data=output)

@app.route('/la_areas/<la_code>')
def la_areas(la_code):
    data = {'la_code': la_code}
    sql = '''
    SELECT
    s.code as scat_code,
    count,
    total_m2,
    total_value,
    total_area_price
    FROM s_vao_base_areas t
    LEFT JOIN c_scat s ON s.code = t.scat_code
    WHERE ba_code = :ba_code
    ORDER BY s.desc
    '''
    output = show_result(sql, data=data)
    return render_template('table_output.html', data=output)

@app.route('/scat_areas/')
def scat_areas_list():
    sql = 'SELECT code, "desc" FROM c_scat ORDER BY "desc"'
    output = show_result(sql)
    output['links'][1] = ('scat_areas', 'scat_code', 0)
    return render_template('table_output.html', data=output)

@app.route('/scat_areas/<scat_code>')
def scat_areas(scat_code):
    data = {'scat_code': scat_code}
    sql = '''
    SELECT
    s.desc,
    count,
    total_m2,
    total_value,
    total_area_price
    FROM s_vao_base_areas t
    LEFT JOIN c_ba s ON s.code = t.ba_code
    WHERE scat_code = :scat_code
    ORDER BY s.desc
    '''
    output = show_result(sql, data=data)
    return render_template('table_output.html', data=output)

@app.route('/scat_diff/')
def scat_diff():
    sql = '''
    SELECT l.uarn, l.scat_code, b.scat_code, l.la_code
    FROM vao_list l
    LEFT JOIN v_vao_base b ON b.uarn = l.uarn
    WHERE l.scat_code != b.scat_code
    '''
    output = show_result(sql)
    return render_template('table_output.html', data=output)

@app.route('/premises/<uarn>')
def premises(uarn):
    data = {'uarn': uarn}
    output = []

    tables = [
        'vao_list',
        'v_vao_base',
        'vao_line',
        'vao_additions',
        'vao_plant',
        'vao_parking',
        'vao_adj',
        'vao_adj_totals',
    ]

    single_row_tables = [
        'vao_list',
        'v_vao_base',
    ]

    for table in tables:
        sql = 'SELECT * FROM %s WHERE uarn = :uarn' % table
        out = show_result(sql, table, data=data)
        out['offset'] = ''
        out['title'] = table
        output.append((out, table in single_row_tables))

    return render_template('premises.html', output=output)

@app.route('/postcode/')
def postcode_premises():
    outcode = None
    postcode = request.args.get('postcode')
    sql = None
    if postcode:
        postcode = postcode.upper().strip()
        if re.match('^[A-Z]{1,2}\d{1,2}[A-Z]?$', postcode):
            outcode = postcode
        else:
            outcode = None
    if outcode:
        print('outcode %s' % outcode)
        data = {'outcode': outcode}
        sql = '''
        SELECT v.uarn uarn, v.pc, v.town, b.uarn summary, v.scat_code
        FROM vao_list v
        LEFT OUTER JOIN v_vao_base b ON b.uarn = v.uarn
        LEFT JOIN c_scat s ON s.code = v.scat_code
        WHERE v.outcode = :outcode
        ORDER BY s.desc, v.pc
        '''
    elif postcode:
        print('postcode %s' % postcode)
        data = {'postcode': postcode.replace(' ', '')}
        sql = '''
        SELECT v.uarn uarn, v.pc, v.town, b.uarn summary, v.scat_code
        FROM vao_list v
        LEFT OUTER JOIN v_vao_base b ON b.uarn = v.uarn
        LEFT JOIN c_scat s ON s.code = v.scat_code
        WHERE v.pcc = :postcode
        ORDER BY s.desc, v.pc
        '''
    if sql:
        output = show_result(sql, data=data)
        output['functions'][3] = (add_yes,)
    else:
        output = None
        postcode = ''
    return render_template('postcode.html', data=output, postcode=postcode)
<commit_msg>bug fix scat areas<commit_after>import re
from numbers import Number
import urllib

from sqlalchemy.engine import reflection

from flask import Flask, render_template, url_for, abort, request, escape, Markup
from flask.ext.sqlalchemy import SQLAlchemy
from werkzeug import url_encode

import sa_common
import config

app = Flask(__name__)
app.config['SQLALCHEMY_DATABASE_URI'] = config.CONNECTION_STRING
app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False
db = SQLAlchemy(app)

def html_format(html, *args, **kw):
    # escape any values
    args = [escape(arg) for arg in args]
    kw = {k: escape(v) for k, v in kw.items()}
    return Markup(unicode(html).format(*args, **kw))

def add_yes(arg):
    if arg:
        return 'YES'
    return ''

def make_html_span(name, cls):
    return '{{{name}}}'.format(name=name, cls=cls)

def code_desc(arg, code_type):
    codes = codes_data.get(code_type)
    if not codes:
        return arg
    if arg:
        desc = codes.get(arg)
        html = make_html_span('code', 'code') + ' '
        if desc:
            html += make_html_span('desc', 'code_desc')
        else:
            html += make_html_span('desc', 'code_unknown')
            desc = 'UNKNOWN'
        return html_format(html, code=arg, desc=desc)

    html = make_html_span('code', 'code_missing')
    return html_format(html, code='MISSING')

@app.template_global()
def modify_query(**new_values):
    args = request.args.copy()

    for key, value in new_values.items():
        args[key] = value

    return '{}?{}'.format(urllib.quote(request.path), url_encode(args))

@app.template_global()
def cell_function(info, value):
    if len(info) == 1:
        return info[0](value)
    return info[0](value, *info[1:])

@app.template_global()
def make_td_class(arg):
    if isinstance(arg, Number):
        return Markup(' class="numeric"')
    return ''

def run_sql(*args, **kw):
    return sa_common.run_sql(db.engine, *args, **kw)

def get_indexes(*args, **kw):
    return sa_common.get_indexes(db.engine, *args, **kw)

def get_primary_keys(*args, **kw):
    return sa_common.get_primary_keys(db.engine, *args, **kw)

def table_list(*args, **kw):
    return sa_common.table_list(db.engine, *args, **kw)

codes_data = {}
for table in table_list():
    if table.startswith('c_'):
        codes = {}
        sql = 'SELECT code, "desc" FROM "{table}"'.format(table=table)
        result = run_sql(sql)
        for row in result:
            codes[row[0]] = row[1]
        codes_data[table[2:]] = codes

def auto_links(fields):
    links = {}
    for i, field in enumerate(fields):
        if field.get('name') == 'uarn':
            links[i] = ('premises', 'uarn', i)
    return links

def auto_functions(fields):
    functions = {}
    for i, field in enumerate(fields):
        if field.get('name').endswith('_code'):
            functions[i] = (code_desc, field.get('name')[:-5])
    return functions

def show_result(sql, table=None, data=None):
    offset = int(request.args.get('offset', 0))
    # We need to have a result to get the field types
    if data is None:
        data = {}
    result = run_sql(sql + ' LIMIT 1', data)
    fields = sa_common.get_result_fields(db.engine, result, table)
    # Now run query
    result = run_sql(sql + (' LIMIT 1000 OFFSET %s' % offset), data)
    output = {
        'fields': fields,
        'data': result,
        'offset': offset,
    }
    if not 'raw' in request.args:
        output['links'] = auto_links(fields)
        output['functions'] = auto_functions(fields)
    return output

def show_table(table):
    sql = 'SELECT * FROM "%s"' % table
    return show_result(sql, table)

@app.route('/')
def home():
    return render_template('home.html')

@app.route('/table/')
def tables():
    out = []
    for table in sorted(table_list()):
        out.append(table)
    return render_template('tables.html', data=out)

@app.route('/table/
')
def table(table=None):
    if table not in table_list():
        abort(404)
    data = show_table(table)
    data['title'] = table

    return render_template('table_output.html', data=data)

@app.route('/la/')
def la_list():
    sql = '''
    SELECT code, "desc" FROM c_la
    WHERE LEFT(code, 1) IN ('E', 'w')
    ORDER BY "desc"
    '''
    output = show_result(sql)
    output['links'][1] = ('la_premises_list', 'la_code', 0)
    return render_template('table_output.html', data=output)

@app.route('/la/<la_code>')
def la_premises_list(la_code):
    data = {'la_code': la_code}
    sql = '''
    SELECT v.uarn, b.uarn, s.code as scat_code
    FROM vao_list v
    LEFT OUTER JOIN v_vao_base b
    ON b.uarn = v.uarn
    LEFT JOIN c_scat s ON s.code = v.scat_code
    WHERE v.la_code = :la_code
    ORDER BY s.desc
    '''
    output = show_result(sql, data=data)
    del output['links'][1]
    output['functions'][1] = (add_yes,)
    output['fields'][1]['name'] = 'summary'
    return render_template('table_output.html', data=output)

@app.route('/scat/')
def scat_list():
    sql = 'SELECT code, "desc" FROM c_scat ORDER BY "desc"'
    output = show_result(sql)
    output['links'][1] = ('scat_premises_list', 'scat_code', 0)
    return render_template('table_output.html', data=output)

@app.route('/scat/<scat_code>')
def scat_premises_list(scat_code):
    data = {'scat_code': scat_code}
    sql = '''
    SELECT v.uarn, b.uarn, c.code as ba_code
    FROM vao_list v
    LEFT OUTER JOIN v_vao_base b
    ON b.uarn = v.uarn
    LEFT JOIN c_ba c ON c.code = v.ba_code
    WHERE v.scat_code = :scat_code
    ORDER BY c.desc
    '''
    output = show_result(sql, data=data)
    del output['links'][1]
    output['fields'][1]['name'] = 'summary'
    output['functions'][1] = (add_yes,)
    return render_template('table_output.html', data=output)

@app.route('/la_areas/')
def la_areas_list():
    sql = '''
    SELECT code, "desc" FROM c_la
    WHERE LEFT(code, 1) IN ('E', 'w')
    ORDER BY "desc"
    '''
    output = show_result(sql)
    output['links'][1] = ('la_areas', 'la_code', 0)
    return render_template('table_output.html', data=output)

@app.route('/la_areas/<la_code>')
def la_areas(la_code):
    data = {'la_code': la_code}
    sql = '''
    SELECT
    s.code as scat_code,
    count,
    total_m2,
    total_value,
    total_area_price
    FROM s_vao_base_areas t
    LEFT JOIN c_scat s ON s.code = t.scat_code
    WHERE ba_code = :ba_code
    ORDER BY s.desc
    '''
    output = show_result(sql, data=data)
    return render_template('table_output.html', data=output)

@app.route('/scat_areas/')
def scat_areas_list():
    sql = 'SELECT code, "desc" FROM c_scat ORDER BY "desc"'
    output = show_result(sql)
    output['links'][1] = ('scat_areas', 'scat_code', 0)
    return render_template('table_output.html', data=output)

@app.route('/scat_areas/<scat_code>')
def scat_areas(scat_code):
    data = {'scat_code': scat_code}
    sql = '''
    SELECT
    s.desc,
    count,
    total_m2,
    total_value,
    total_area_price
    FROM s_vao_base_areas t
    LEFT JOIN c_la s ON s.code = t.la_code
    WHERE scat_code = :scat_code
    ORDER BY s.desc
    '''
    output = show_result(sql, data=data)
    return render_template('table_output.html', data=output)

@app.route('/scat_diff/')
def scat_diff():
    sql = '''
    SELECT l.uarn, l.scat_code, b.scat_code, l.la_code
    FROM vao_list l
    LEFT JOIN v_vao_base b ON b.uarn = l.uarn
    WHERE l.scat_code != b.scat_code
    '''
    output = show_result(sql)
    return render_template('table_output.html', data=output)

@app.route('/premises/<uarn>')
def premises(uarn):
    data = {'uarn': uarn}
    output = []

    tables = [
        'vao_list',
        'v_vao_base',
        'vao_line',
        'vao_additions',
        'vao_plant',
        'vao_parking',
        'vao_adj',
        'vao_adj_totals',
    ]

    single_row_tables = [
        'vao_list',
        'v_vao_base',
    ]

    for table in tables:
        sql = 'SELECT * FROM %s WHERE uarn = :uarn' % table
        out = show_result(sql, table, data=data)
        out['offset'] = ''
        out['title'] = table
        output.append((out, table in single_row_tables))

    return render_template('premises.html', output=output)

@app.route('/postcode/')
def postcode_premises():
    outcode = None
    postcode = request.args.get('postcode')
    sql = None
    if postcode:
        postcode = postcode.upper().strip()
        if re.match('^[A-Z]{1,2}\d{1,2}[A-Z]?$', postcode):
            outcode = postcode
        else:
            outcode = None
    if outcode:
        print('outcode %s' % outcode)
        data = {'outcode': outcode}
        sql = '''
        SELECT v.uarn uarn, v.pc, v.town, b.uarn summary, v.scat_code
        FROM vao_list v
        LEFT OUTER JOIN v_vao_base b ON b.uarn = v.uarn
        LEFT JOIN c_scat s ON s.code = v.scat_code
        WHERE v.outcode = :outcode
        ORDER BY s.desc, v.pc
        '''
    elif postcode:
        print('postcode %s' % postcode)
        data = {'postcode': postcode.replace(' ', '')}
        sql = '''
        SELECT v.uarn uarn, v.pc, v.town, b.uarn summary, v.scat_code
        FROM vao_list v
        LEFT OUTER JOIN v_vao_base b ON b.uarn = v.uarn
        LEFT JOIN c_scat s ON s.code = v.scat_code
        WHERE v.pcc = :postcode
        ORDER BY s.desc, v.pc
        '''
    if sql:
        output = show_result(sql, data=data)
        output['functions'][3] = (add_yes,)
    else:
        output = None
        postcode = ''
    return render_template('postcode.html', data=output, postcode=postcode)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

# As per http://blog.rtwilson.com/how-to-make-your-sphinx-documentation-compile-with-readthedocs-when-youre-using-numpy-and-scipy/ :
import sys, os

from mock import Mock

MOCK_MODULES = [
    'fitsio', 'astrometry', 'astrometry.util',
    'astrometry.util.fits', 'astrometry.util.plotutils',
    'astrometry.util.resample', 'astrometry.util.ttime',
    'astrometry.util.starutil_numpy', 'astrometry.util.miscutils',
    'astrometry.util.util', 'astrometry.util.multiproc',
    'astrometry.util.file',
    'astrometry.libkd',
    'tractor',
    'tractor.ellipses', 'tractor.galaxy', 'tractor.utils', 'tractor.basics',
    'tractor.engine', 'tractor.psfex', 'tractor.sfd',
    'utils', 'utils.debugpool',
    'numpy',
    'scipy',
    'scipy.interpolate', 'scipy.ndimage', 'scipy.ndimage.filters',
    'scipy.ndimage.measurements', 'scipy.ndimage.morphology',
    'matplotlib', 'pylab', 'matplotlib.pyplot',
    ]
for mod_name in MOCK_MODULES:
    sys.modules[mod_name] = Mock()

#
# legacypipe documentation build configuration file, created by
# sphinx-quickstart on Fri Jul 10 15:38:03 2015.
#
# This file is execfile()d with the current directory set to its containing dir.
#
# Note that not all possible configuration values are present in this
# autogenerated file.
#
# All configuration values have a default; values that are commented out
# serve to show the default.

# If extensions (or modules to document with autodoc) are in another directory,
# add these directories to sys.path here. If the directory is relative to the
# documentation root, use os.path.abspath to make it absolute, like shown here.
sys.path.insert(0, os.path.abspath('../py'))

# -- General configuration -----------------------------------------------------

# If your documentation needs a minimal Sphinx version, state it here.
#needs_sphinx = '1.0'

# Add any Sphinx extension module names here, as strings. They can be extensions
# coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
extensions = ['sphinx.ext.autodoc', 'sphinx.ext.doctest', 'sphinx.ext.coverage',
              'sphinx.ext.pngmath', 'sphinx.ext.mathjax', 'sphinx.ext.viewcode',
              'numpydoc']

# Add any paths that contain templates here, relative to this directory.
templates_path = ['_templates']

# The suffix of source filenames.
source_suffix = '.rst'

# The encoding of source files.
#source_encoding = 'utf-8-sig'

# The master toctree document.
master_doc = 'index'

# General information about the project.
project = u'legacypipe'
copyright = u'2015, LegacySurvey.org'

# The version info for the project you're documenting, acts as replacement for
# |version| and |release|, also used in various other places throughout the
# built documents.
#
# The short X.Y version.
version = '1.0'
# The full version, including alpha/beta/rc tags.
release = '1.0'

# The language for content autogenerated by Sphinx. Refer to documentation
# for a list of supported languages.
#language = None

# There are two options for replacing |today|: either, you set today to some
# non-false value, then it is used:
#today = ''
# Else, today_fmt is used as the format for a strftime call.
#today_fmt = '%B %d, %Y'

# List of patterns, relative to source directory, that match files and
# directories to ignore when looking for source files.
exclude_patterns = ['_build']

# The reST default role (used for this markup: `text`) to use for all documents.
#default_role = None

# If true, '()' will be appended to :func: etc. cross-reference text.
#add_function_parentheses = True

# If true, the current module name will be prepended to all description
# unit titles (such as .. function::).
#add_module_names = True

# If true, sectionauthor and moduleauthor directives will be shown in the
# output. They are ignored by default.
#show_authors = False

# The name of the Pygments (syntax highlighting) style to use.
pygments_style = 'sphinx'

# A list of ignored prefixes for module index sorting.
#modindex_common_prefix = []

# -- Options for HTML output ---------------------------------------------------

# The theme to use for HTML and HTML Help pages.  See the documentation for
# a list of builtin themes.
html_theme = 'default'

# Theme options are theme-specific and customize the look and feel of a theme
# further.  For a list of options available for each theme, see the
# documentation.
#html_theme_options = {}

# Add any paths that contain custom themes here, relative to this directory.
#html_theme_path = []

# The name for this set of Sphinx documents.  If None, it defaults to
# "<project> v<release> documentation".
#html_title = None

# A shorter title for the navigation bar.  Default is the same as html_title.
#html_short_title = None

# The name of an image file (relative to this directory) to place at the top
# of the sidebar.
html_logo = 'logo.png'

# The name of an image file (within the static path) to use as favicon of the
# docs.  This file should be a Windows icon file (.ico) being 16x16 or 32x32
# pixels large.
html_favicon = 'favicon.ico'

# Add any paths that contain custom static files (such as style sheets) here,
# relative to this directory. They are copied after the builtin static files,
# so a file named "default.css" will overwrite the builtin "default.css".
html_static_path = ['_static']

# If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
# using the given strftime format.
#html_last_updated_fmt = '%b %d, %Y'

# If true, SmartyPants will be used to convert quotes and dashes to
# typographically correct entities.
#html_use_smartypants = True

# Custom sidebar templates, maps document names to template names.
#html_sidebars = {}

# Additional templates that should be rendered to pages, maps page names to
# template names.
#html_additional_pages = {}

# If false, no module index is generated.
#html_domain_indices = True

# If false, no index is generated.
#html_use_index = True

# If true, the index is split into individual pages for each letter.
#html_split_index = False

# If true, links to the reST sources are added to the pages.
#html_show_sourcelink = True

# If true, "Created using Sphinx" is shown in the HTML footer. Default is True.
#html_show_sphinx = True

# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True.
#html_show_copyright = True

# If true, an OpenSearch description file will be output, and all pages will
# contain a <link> tag referring to it.  The value of this option must be the
# base URL from which the finished HTML is served.
#html_use_opensearch = ''

# This is the file name suffix for HTML files (e.g. ".xhtml").
#html_file_suffix = None

# Output file base name for HTML help builder.
htmlhelp_basename = 'legacypipedoc'

# -- Options for LaTeX output --------------------------------------------------

latex_elements = {
# The paper size ('letterpaper' or 'a4paper').
#'papersize': 'letterpaper',

# The font size ('10pt', '11pt' or '12pt').
#'pointsize': '10pt',

# Additional stuff for the LaTeX preamble.
#'preamble': '',
}

# Grouping the document tree into LaTeX files. List of tuples
# (source start file, target name, title, author, documentclass [howto/manual]).
latex_documents = [
  ('index', 'legacypipe.tex', u'legacypipe Documentation',
   u'LegacySurvey.org', 'manual'),
]

# The name of an image file (relative to this directory) to place at the top of
# the title page.
#latex_logo = None

# For "manual" documents, if this is true, then toplevel headings are parts,
# not chapters.
#latex_use_parts = False

# If true, show page references after internal links.
#latex_show_pagerefs = False

# If true, show URL addresses after external links.
#latex_show_urls = False

# Documents to append as an appendix to all manuals.
#latex_appendices = []

# If false, no module index is generated.
#latex_domain_indices = True

# -- Options for manual page output --------------------------------------------

# One entry per manual page. List of tuples
# (source start file, name, description, authors, manual section).
man_pages = [
    ('index', 'legacypipe', u'legacypipe Documentation',
     [u'LegacySurvey.org'], 1)
]

# If true, show URL addresses after external links.
#man_show_urls = False

# -- Options for Texinfo output ------------------------------------------------

# Grouping the document tree into Texinfo files. List of tuples
# (source start file, target name, title, author,
#  dir menu entry, description, category)
texinfo_documents = [
  ('index', 'legacypipe', u'legacypipe Documentation',
   u'LegacySurvey.org', 'legacypipe', 'One line description of project.',
   'Miscellaneous'),
]

# Documents to append as an appendix to all manuals.
#texinfo_appendices = []

# If false, no module index is generated.
#texinfo_domain_indices = True

# How to display URL addresses: 'footnote', 'no', or 'inline'.
#texinfo_show_urls = 'footnote'
<commit_msg>fake astrometry.util.timingpool in doc build<commit_after># -*- coding: utf-8 -*-

# As per http://blog.rtwilson.com/how-to-make-your-sphinx-documentation-compile-with-readthedocs-when-youre-using-numpy-and-scipy/ :
import sys, os

from mock import Mock

MOCK_MODULES = [
    'fitsio', 'astrometry', 'astrometry.util',
    'astrometry.util.fits', 'astrometry.util.plotutils',
    'astrometry.util.resample', 'astrometry.util.ttime',
    'astrometry.util.starutil_numpy', 'astrometry.util.miscutils',
    'astrometry.util.util', 'astrometry.util.multiproc',
    'astrometry.util.file', 'astrometry.util.timingpool',
    'astrometry.libkd',
    'tractor',
    'tractor.ellipses', 'tractor.galaxy', 'tractor.utils', 'tractor.basics',
    'tractor.engine', 'tractor.psfex', 'tractor.sfd',
    'utils', 'utils.debugpool',
    'numpy',
    'scipy',
    'scipy.interpolate', 'scipy.ndimage', 'scipy.ndimage.filters',
    'scipy.ndimage.measurements', 'scipy.ndimage.morphology',
    'matplotlib', 'pylab', 'matplotlib.pyplot',
    ]
for mod_name in MOCK_MODULES:
    sys.modules[mod_name] = Mock()

#
# legacypipe documentation build configuration file, created by
# sphinx-quickstart on Fri Jul 10 15:38:03 2015.
#
# This file is execfile()d with the current directory set to its containing dir.
#
# Note that not all possible configuration values are present in this
# autogenerated file.
#
# All configuration values have a default; values that are commented out
# serve to show the default.

# If extensions (or modules to document with autodoc) are in another directory,
# add these directories to sys.path here. If the directory is relative to the
# documentation root, use os.path.abspath to make it absolute, like shown here.
sys.path.insert(0, os.path.abspath('../py'))

# -- General configuration -----------------------------------------------------

# If your documentation needs a minimal Sphinx version, state it here.
#needs_sphinx = '1.0'

# Add any Sphinx extension module names here, as strings. They can be extensions
# coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
extensions = ['sphinx.ext.autodoc', 'sphinx.ext.doctest', 'sphinx.ext.coverage',
              'sphinx.ext.pngmath', 'sphinx.ext.mathjax', 'sphinx.ext.viewcode',
              'numpydoc']

# Add any paths that contain templates here, relative to this directory.
templates_path = ['_templates']

# The suffix of source filenames.
source_suffix = '.rst'

# The encoding of source files.
#source_encoding = 'utf-8-sig'

# The master toctree document.
master_doc = 'index'

# General information about the project.
project = u'legacypipe'
copyright = u'2015, LegacySurvey.org'

# The version info for the project you're documenting, acts as replacement for
# |version| and |release|, also used in various other places throughout the
# built documents.
#
# The short X.Y version.
version = '1.0'
# The full version, including alpha/beta/rc tags.
release = '1.0'

# The language for content autogenerated by Sphinx. Refer to documentation
# for a list of supported languages.
#language = None

# There are two options for replacing |today|: either, you set today to some
# non-false value, then it is used:
#today = ''
# Else, today_fmt is used as the format for a strftime call.
#today_fmt = '%B %d, %Y'

# List of patterns, relative to source directory, that match files and
# directories to ignore when looking for source files.
exclude_patterns = ['_build']

# The reST default role (used for this markup: `text`) to use for all documents.
#default_role = None

# If true, '()' will be appended to :func: etc. cross-reference text.
#add_function_parentheses = True

# If true, the current module name will be prepended to all description
# unit titles (such as .. function::).
#add_module_names = True

# If true, sectionauthor and moduleauthor directives will be shown in the
# output. They are ignored by default.
#show_authors = False

# The name of the Pygments (syntax highlighting) style to use.
pygments_style = 'sphinx'

# A list of ignored prefixes for module index sorting.
#modindex_common_prefix = []

# -- Options for HTML output ---------------------------------------------------

# The theme to use for HTML and HTML Help pages.  See the documentation for
# a list of builtin themes.
html_theme = 'default'

# Theme options are theme-specific and customize the look and feel of a theme
# further.  For a list of options available for each theme, see the
# documentation.
#html_theme_options = {}

# Add any paths that contain custom themes here, relative to this directory.
#html_theme_path = []

# The name for this set of Sphinx documents.  If None, it defaults to
# "<project> v<release> documentation".
#html_title = None

# A shorter title for the navigation bar.  Default is the same as html_title.
#html_short_title = None

# The name of an image file (relative to this directory) to place at the top
# of the sidebar.
html_logo = 'logo.png'

# The name of an image file (within the static path) to use as favicon of the
# docs.  This file should be a Windows icon file (.ico) being 16x16 or 32x32
# pixels large.
html_favicon = 'favicon.ico'

# Add any paths that contain custom static files (such as style sheets) here,
# relative to this directory. They are copied after the builtin static files,
# so a file named "default.css" will overwrite the builtin "default.css".
html_static_path = ['_static']

# If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
# using the given strftime format.
#html_last_updated_fmt = '%b %d, %Y'

# If true, SmartyPants will be used to convert quotes and dashes to
# typographically correct entities.
#html_use_smartypants = True

# Custom sidebar templates, maps document names to template names.
#html_sidebars = {}

# Additional templates that should be rendered to pages, maps page names to
# template names.
#html_additional_pages = {}

# If false, no module index is generated.
#html_domain_indices = True

# If false, no index is generated.
#html_use_index = True

# If true, the index is split into individual pages for each letter.
#html_split_index = False

# If true, links to the reST sources are added to the pages.
#html_show_sourcelink = True

# If true, "Created using Sphinx" is shown in the HTML footer. Default is True.
#html_show_sphinx = True

# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True.
#html_show_copyright = True

# If true, an OpenSearch description file will be output, and all pages will
# contain a <link> tag referring to it.  The value of this option must be the
# base URL from which the finished HTML is served.
#html_use_opensearch = ''

# This is the file name suffix for HTML files (e.g. ".xhtml").
#html_file_suffix = None

# Output file base name for HTML help builder.
htmlhelp_basename = 'legacypipedoc'

# -- Options for LaTeX output --------------------------------------------------

latex_elements = {
# The paper size ('letterpaper' or 'a4paper').
#'papersize': 'letterpaper',

# The font size ('10pt', '11pt' or '12pt').
#'pointsize': '10pt',

# Additional stuff for the LaTeX preamble.
#'preamble': '',
}

# Grouping the document tree into LaTeX files. List of tuples
# (source start file, target name, title, author, documentclass [howto/manual]).
latex_documents = [
  ('index', 'legacypipe.tex', u'legacypipe Documentation',
   u'LegacySurvey.org', 'manual'),
]

# The name of an image file (relative to this directory) to place at the top of
# the title page.
#latex_logo = None

# For "manual" documents, if this is true, then toplevel headings are parts,
# not chapters.
#latex_use_parts = False

# If true, show page references after internal links.
#latex_show_pagerefs = False

# If true, show URL addresses after external links.
#latex_show_urls = False

# Documents to append as an appendix to all manuals.
#latex_appendices = []

# If false, no module index is generated.
#latex_domain_indices = True

# -- Options for manual page output --------------------------------------------

# One entry per manual page. List of tuples
# (source start file, name, description, authors, manual section).
man_pages = [
    ('index', 'legacypipe', u'legacypipe Documentation',
     [u'LegacySurvey.org'], 1)
]

# If true, show URL addresses after external links.
#man_show_urls = False

# -- Options for Texinfo output ------------------------------------------------

# Grouping the document tree into Texinfo files. List of tuples
# (source start file, target name, title, author,
#  dir menu entry, description, category)
texinfo_documents = [
  ('index', 'legacypipe', u'legacypipe Documentation',
   u'LegacySurvey.org', 'legacypipe', 'One line description of project.',
   'Miscellaneous'),
]

# Documents to append as an appendix to all manuals.
#texinfo_appendices = []

# If false, no module index is generated.
#texinfo_domain_indices = True

# How to display URL addresses: 'footnote', 'no', or 'inline'.
#texinfo_show_urls = 'footnote'
<|endoftext|>


<commit_before>import abc
from nalaf.structures.data import Edge
from itertools import product, chain

class EdgeGenerator(object):
    """
    Abstract class for generating edges between two entities. Each edge represents
    a possible relationship between the two entities
    Subclasses that inherit this class should:
    * Be named [Name]EdgeGenerator
    * Implement the abstract method generate
    * Append new items to the list field "edges" of each Part in the dataset
    """

    def __init__(self, entity1_class, entity2_class, relation_type):
        self.entity1_class = entity1_class
        self.entity2_class = entity2_class
        self.relation_type = relation_type

    @abc.abstractmethod
    def generate(self, dataset):
        """
        :type dataset: nalaf.structures.data.Dataset
        """
        return

class SentenceDistanceEdgeGenerator(EdgeGenerator):
    """
    Simple implementation of generating edges between the two entities
    if they are #`distance` sentences away (always same part).

    As a special case, if `distance` is None, the distance is disregarded,
    consequently creating edges for all entities within a part (paragraph or what not).
    """

    def __init__(self, entity1_class, entity2_class, relation_type, distance, use_predicted_entities=True, rewrite_edges=True):
        # Note: would be nice to implement the word filter too here -- see below

        super().__init__(entity1_class, entity2_class, relation_type)
        self.distance = distance
        self.use_predicted_entities = use_predicted_entities
        self.rewrite_edges = rewrite_edges

        self.part_entities = (lambda part: chain(part.annotations, part.predicted_annotations) if self.use_predicted_entities else part.annotations)

    def generate(self, dataset):

        for part in dataset.parts():
            part.edges = []

            for e_1, e_2 in product(
                    (e for e in self.part_entities(part) if e.class_id == self.entity1_class),
                    (e for e in self.part_entities(part) if e.class_id == self.entity2_class)):

                s1_index = part.get_sentence_index_for_annotation(e_1)
                s2_index = part.get_sentence_index_for_annotation(e_2)

                if s2_index < s1_index:
                    s1_index, s2_index = s2_index, s1_index

                if e_2.offset < e_1.offset:
                    e_1, e_2 = e_2, e_1

                pair_distance = s2_index - s1_index
                assert pair_distance >= 0  # Because they must be sorted

                if pair_distance == self.distance or self.distance is None:
                    edge = Edge(self.relation_type, e_1, e_2, part, part, s1_index, s2_index)
                    part.edges.append(edge)

class WordFilterEdgeGenerator(EdgeGenerator):
    """
    Simple implementation of generating edges between the two entities
    if they are contained in the same sentence AND the sentence
    contains one of the trigger-like given words

    **It only uses the _gold_ annotations**

    """

    def __init__(self, entity1_class, entity2_class, relation_type, words):
        super().__init__(entity1_class, entity2_class, relation_type)
        self.words = words

    def generate(self, dataset):
        from itertools import product

        for part in dataset.parts():
            part.edges = []

            for ann_1, ann_2 in product(
                    (ann for ann in part.annotations if ann.class_id == self.entity1_class),
                    (ann for ann in part.annotations if ann.class_id == self.entity2_class)):

                index_1 = part.get_sentence_index_for_annotation(ann_1)
                index_2 = part.get_sentence_index_for_annotation(ann_2)

                if index_1 == index_2 and index_1 is not None:

                    for token in part.sentences[index_1]:

                        if token.word in self.words:
                            edge = Edge(self.relation_type, ann_1, ann_2, part, part, index_1, index_2)

                            part.edges.append(edge)

                            break
<commit_msg>make possible not to rewrite edges<commit_after>import abc
from nalaf.structures.data import Edge
from itertools import product, chain

class EdgeGenerator(object):
    """
    Abstract class for generating edges between two entities. Each edge represents
    a possible relationship between the two entities
    Subclasses that inherit this class should:
    * Be named [Name]EdgeGenerator
    * Implement the abstract method generate
    * Append new items to the list field "edges" of each Part in the dataset
    """

    def __init__(self, entity1_class, entity2_class, relation_type):
        self.entity1_class = entity1_class
        self.entity2_class = entity2_class
        self.relation_type = relation_type

    @abc.abstractmethod
    def generate(self, dataset):
        """
        :type dataset: nalaf.structures.data.Dataset
        """
        return

class SentenceDistanceEdgeGenerator(EdgeGenerator):
    """
    Simple implementation of generating edges between the two entities
    if they are #`distance` sentences away (always same part).

    As a special case, if `distance` is None, the distance is disregarded,
    consequently creating edges for all entities within a part (paragraph or what not).
    """

    def __init__(self, entity1_class, entity2_class, relation_type, distance, use_predicted_entities=True, rewrite_edges=True):
        # Note: would be nice to implement the word filter too here -- see below

        super().__init__(entity1_class, entity2_class, relation_type)
        self.distance = distance
        self.use_predicted_entities = use_predicted_entities
        self.rewrite_edges = rewrite_edges

        self.part_entities = (lambda part: chain(part.annotations, part.predicted_annotations) if self.use_predicted_entities else part.annotations)

    def generate(self, dataset):

        for part in dataset.parts():
            if self.rewrite_edges:
                part.edges = []

            for e_1, e_2 in product(
                    (e for e in self.part_entities(part) if e.class_id == self.entity1_class),
                    (e for e in self.part_entities(part) if e.class_id == self.entity2_class)):

                s1_index = part.get_sentence_index_for_annotation(e_1)
                s2_index = part.get_sentence_index_for_annotation(e_2)

                if s2_index < s1_index:
                    s1_index, s2_index = s2_index, s1_index

                if e_2.offset < e_1.offset:
                    e_1, e_2 = e_2, e_1

                pair_distance = s2_index - s1_index
                assert pair_distance >= 0  # Because they must be sorted

                if pair_distance == self.distance or self.distance is None:
                    edge = Edge(self.relation_type, e_1, e_2, part, part, s1_index, s2_index)
                    part.edges.append(edge)

class WordFilterEdgeGenerator(EdgeGenerator):
    """
    Simple implementation of generating edges between the two entities
    if they are contained in the same sentence AND the sentence
    contains one of the trigger-like given words

    **It only uses the _gold_ annotations**

    """

    def __init__(self, entity1_class, entity2_class, relation_type, words):
        super().__init__(entity1_class, entity2_class, relation_type)
        self.words = words

    def generate(self, dataset):
        from itertools import product

        for part in dataset.parts():
            part.edges = []

            for ann_1, ann_2 in product(
                    (ann for ann in part.annotations if ann.class_id == self.entity1_class),
                    (ann for ann in part.annotations if ann.class_id == self.entity2_class)):

                index_1 = part.get_sentence_index_for_annotation(ann_1)
                index_2 = part.get_sentence_index_for_annotation(ann_2)

                if index_1 == index_2 and index_1 is not None:

                    for token in part.sentences[index_1]:

                        if token.word in self.words:
                            edge = Edge(self.relation_type, ann_1, ann_2, part, part, index_1, index_2)

                            part.edges.append(edge)

                            break
<|endoftext|>