<commit_before>import os
import sys
import argparse
import json

# Global variables

base_dir = '/Users/monad/Work/data'
cult10m_dir = os.path.join(base_dir, '10m_cultural', '10m_cultural')
phys10m_dir = os.path.join(base_dir, '10m_physical')
cult50m_dir = os.path.join(base_dir, '50m_cultural')
phys50m_dir = os.path.join(base_dir, '50m_physical')
edited50m_dir = os.path.join(base_dir, 'edited50m')

land_file = os.path.join(phys50m_dir, 'ne_50m_land.shp')
land_boundaries_file = os.path.join(phys50m_dir, 'ne_50m_land.shp')
boundaries_file = os.path.join(cult50m_dir, 'ne_50m_admin_0_boundary_lines_land.shp')
countries_file = os.path.join(cult50m_dir, 'ne_50m_admin_0_countries.shp')
tiny_file = os.path.join(cult50m_dir, 'ne_50m_admin_0_tiny_countries.shp')
lakes_file = os.path.join(phys50m_dir, 'ne_50m_lakes.shp')
disputed_file = os.path.join(cult50m_dir, 'ne_50m_admin_0_breakaway_disputed_areas.shp')

land_file = os.path.join(phys10m_dir, 'ne_10m_land.shp')
#land_file = os.path.join(cult10m_dir, 'ne_10m_admin_0_sovereignty.shp')
land_boundaries_file = os.path.join(phys10m_dir, 'ne_10m_land.shp')
#boundaries_file = os.path.join(cult10m_dir, 'ne_10m_admin_0_boundary_lines_land.shp')
boundaries_file = os.path.join(edited50m_dir, 'ne_50m_admin_0_boundary_lines_land.shp')
#countries_file = os.path.join(cult10m_dir, 'ne_10m_admin_0_countries.shp')
#tiny_file = os.path.join(cult10m_dir, 'ne_10m_admin_0_tiny_countries.shp')
#lakes_file = os.path.join(phys10m_dir, 'ne_10m_lakes.shp')

lakes_size = 3

def report_error(msg):
    sys.stderr.write("**ERROR**: {0}\n".format(msg))

# Command line arguments

parser = argparse.ArgumentParser(description="Creates a map from a data file")

size_group = parser.add_mutually_exclusive_group()
size_group.add_argument('--size', nargs=2, metavar=('W', 'H'),
                        type=int, default=None,
                        help="the size of output images")
size_group.add_argument('--xd', action='store_true',
                        help="xd size (should be set in the input data file)")
size_group.add_argument('--hd', action='store_true',
                        help="0.5 * (xd size)")
size_group.add_argument('--sd', action='store_true',
                        help="0.25 * (xd size)")

parser.add_argument('--png8', action='store_true',
                    help="8-bit PNG images")

parser.add_argument('--out', metavar='DIR',
                    help="the output directory")

parser.add_argument('--color', default='red',
                    help="polygon fill color for countries (use 'none' for no color)")

#parser.add_argument('--line-color', default='black',
#                    help="border line color (use 'none' for no borders)")

parser.add_argument('--scale', type=float, default=1.0,
                    help="scale for lines")

parser.add_argument('--no-markers', action='store_true',
                    help="do not draw markers")

parser.add_argument('--test', action='store_true',
                    help="produce one map only")

parser.add_argument('input_file',
                    help="input JSON data file")

# Parse arguments and load the input file

args = parser.parse_args()

with open(args.input_file) as f:
    data = json.load(f)

class CountryInfo:
    def __init__(self, data):
        self.disputed = None
        self.one_color = False
        self.marker_size = None
        if isinstance(data, unicode):
            self.name = data.encode()
        elif isinstance(data, str):
            self.name = data
        else:
            assert(isinstance(data, dict))
            self.name = data['name'].encode()
            if 'disputed' in data:
                self.disputed = data['disputed'].encode()
            if 'one-color' in data:
                self.one_color = data['one-color']
            if 'marker-size' in data:
                self.marker_size = tuple(data['marker-size'])

countries = []
for country in data['countries']:
    countries.append(CountryInfo(country))

# Validate data

if 'xd-size' not in data:
    report_error("'xd-size' is not defined in {0}".format(args.input_file))
    exit(2)

if ('proj' not in data) or ('bbox' not in data):
    report_error("'proj' or 'bbox' are not defined in {0}".format(args.input_file))
    exit(2)

xd_width, xd_height = data['xd-size']

# Validate arguments

if args.sd:
    width, height = int(xd_width * 0.25), int(xd_height * 0.25)
    args.scale *= 0.25
elif args.hd:
    width, height = int(xd_width * 0.5), int(xd_height * 0.5)
    args.scale *= 0.5
elif args.xd:
    width, height = xd_width, xd_height
elif args.size:
    width, height = args.size
else:
    width, height = xd_width, xd_height

if args.color == 'none':
    args.color = None

#if args.line_color == 'none':
#    args.line_color = None

if args.scale < 0.01 or args.scale > 10:
    sys.stderr.write("\nBad scale: {0}\n\n".format(args.scale))
    sys.exit(1)

if width < 1 or height < 1 or width > 10000 or height > 10000:
    sys.stderr.write("\nBad image size: {0} x {1}\n\n".format(width, height))
    sys.exit(1)

if not args.out:
    args.out = "out_{0}_{1}_{2}".format(data['name'], width, height)

if not os.path.exists(args.out):
    os.makedirs(args.out)

# Styles and layers

# Land

def add_layer_with_style(m, layer, style, style_name=None):
    if not style_name:
        style_name = layer.name + 'Style'
    m.append_style(style_name, style)
    layer.styles.append(style_name)
    m.layers.append(layer)

def land_style():
    s = Style()
    r = Rule()

    ps = PolygonSymbolizer()
    ps.fill = Color('#f1daa0')
    r.symbols.append(ps)

#    stk = Stroke(Color('#4fadc2'), 5.0)
#    ls = LineSymbolizer(stk)
#    r.symbols.append(ls)

    s.rules.append(r)
    return s

def land_layer():
    ds = Shapefile(file=land_file)
    layer = Layer('Land')
    layer.datasource = ds
    return layer

# Land boundaries

def land_boundaries_style():
    s = Style()
    r = Rule()

    stk = Stroke(Color('#4fadc2'), 1.0)
    ls = LineSymbolizer(stk)
    r.symbols.append(ls)

#    ps = PolygonSymbolizer()
#    ps.fill = Color('red')
#    r.symbols.append(ps)

    s.rules.append(r)
    return s

def land_boundaries_layer():
    ds = Shapefile(file=land_boundaries_file)
    layer = Layer('Land Boundaries')
    layer.datasource = ds
    return layer

# Lakes

def lakes_style():
    s = Style()
    r = Rule()

    r.filter = Expression("[scalerank] <= {0}".format(lakes_size))

    ps = PolygonSymbolizer()
    ps.fill = Color('#b3e2ee')
    r.symbols.append(ps)

    stk = Stroke(Color('#4fadc2'), 1.0)
    ls = LineSymbolizer(stk)
    r.symbols.append(ls)

    s.rules.append(r)
    return s

def lakes_layer():
    ds = Shapefile(file=lakes_file)
    layer = Layer('Lakes')
    layer.datasource = ds
    return layer

# Boundaries of countries

def boundaries_style():
    s = Style()
    r = Rule()

    stk = Stroke()
#    stk.add_dash(8, 4)
#    stk.add_dash(2, 2)
#    stk.add_dash(2, 2)
    stk.color = Color('#808080')
    stk.width = 1.0
    ls = LineSymbolizer(stk)
    r.symbols.append(ls)

    s.rules.append(r)
    return s

def boundaries_layer():
    ds = Shapefile(file=boundaries_file)
    layer = Layer('Boundaries')
    layer.datasource = ds
    return layer

# A country

def country_style(name):
    s = Style()
    r = Rule()

    r.filter = Expression("[admin] = '{0}'".format(name))

    if args.color:
        ps = PolygonSymbolizer()
        ps.fill = Color(args.color)
        r.symbols.append(ps)

    s.rules.append(r)
    return s

def country_layer(name):
    ds = Shapefile(file=countries_file)
    layer = Layer("Country " + name)
    layer.datasource = ds
    return layer

def disputed_style(name, one_color=False):
    s = Style()
    r = Rule()

    r.filter = Expression("[name] = '{0}'".format(name))

    ps = PolygonSymbolizer()
    if one_color and args.color:
        ps.fill = Color(args.color)
    else:
        ps.fill = Color('#f76d50')
    r.symbols.append(ps)

    s.rules.append(r)
    return s

def disputed_layer(name):
    ds = Shapefile(file=disputed_file)
    layer = Layer("Disputed " + name)
    layer.datasource = ds
    return layer

def tiny_style(name, size):
    s = Style()
    r = Rule()

    r.filter = Expression("[name] = '{0}' or [sovereignt] = '{0}'".format(name))

    ms = MarkersSymbolizer()
    ms.fill = Color('red')
    ms.opacity = 0.5
    ms.stroke = Stroke(Color('black'), 0.0)
    ms.width = Expression('{0}'.format(size[0]))
    ms.height = Expression('{0}'.format(size[1]))
    r.symbols.append(ms)

    s.rules.append(r)
    return s

def tiny_layer(name):
    ds = Shapefile(file=tiny_file)
    layer = Layer("Tiny " + name)
    layer.datasource = ds
    return layer

# Base map

def base_map(data, width, height):
    m = Map(width, height, data['proj'].encode())
    m.background = Color('#b3e2ee')
    add_layer_with_style(m, land_layer(),
                         land_style(), 'Land Style')
    add_layer_with_style(m, boundaries_layer(),
                         boundaries_style(), 'Boundaries Style')
    add_layer_with_style(m, lakes_layer(),
                         lakes_style(), 'Lakes Style')
    add_layer_with_style(m, land_boundaries_layer(),
                         land_boundaries_style(), 'Land Boundaries Style')
    m.zoom_to_box(Box2d(*data['bbox']))
    return m

# A map with a country

def set_country(m, info):
    style = country_style(info.name)
    layer = country_layer(info.name)
    style_name = 'Style ' + info.name
    m.append_style(style_name, style)
    layer.styles.append(style_name)

    while len(m.layers) > 4:
        del m.layers[1]

    m.layers[1:1] = layer

    if info.disputed:
        style = disputed_style(info.disputed, one_color=info.one_color)
        layer = disputed_layer(info.disputed)
        style_name = 'Disputed Style ' + info.disputed
        m.append_style(style_name, style)
        layer.styles.append(style_name)
        m.layers[2:2] = layer

    if info.marker_size and not args.no_markers:
        style = tiny_style(info.name, info.marker_size)
        layer = tiny_layer(info.name)
        style_name = 'Tiny Style ' + info.name
        m.append_style(style_name, style)
        layer.styles.append(style_name)
        m.layers[2:2] = layer

# The main script

from mapnik import *

out_format = 'png256' if args.png8 else 'png'

m = base_map(data, width, height)
render_to_file(m, os.path.join(args.out, 'a.png'), out_format, args.scale)

if args.test:
    print("done (test)")
    exit(0)

for country in countries:
    print("Processing: {0}".format(country.name))
#    if country.name != 'Mauritius':
#        continue
    set_country(m, country)
    out_name = os.path.join(args.out, "{0}.png".format(country.name))
    render_to_file(m, out_name, out_format, args.scale)

print("done")
<commit_msg>create_map.py: options for 10m and 50m data<commit_after>import os
import sys
import argparse
import json

# Global variables

base_dir = '/Users/monad/Work/data'
cult10m_dir = os.path.join(base_dir, '10m_cultural', '10m_cultural')
phys10m_dir = os.path.join(base_dir, '10m_physical')
cult50m_dir = os.path.join(base_dir, '50m_cultural')
phys50m_dir = os.path.join(base_dir, '50m_physical')
edited50m_dir = os.path.join(base_dir, 'edited50m')

land_file_50m = os.path.join(phys50m_dir, 'ne_50m_land.shp')
land_boundaries_file_50m = os.path.join(phys50m_dir, 'ne_50m_land.shp')
#boundaries_file_50m = os.path.join(cult50m_dir, 'ne_50m_admin_0_boundary_lines_land.shp')
boundaries_file_50m = os.path.join(edited50m_dir, 'ne_50m_admin_0_boundary_lines_land.shp')
countries_file_50m = os.path.join(cult50m_dir, 'ne_50m_admin_0_countries.shp')
lakes_file_50m = os.path.join(phys50m_dir, 'ne_50m_lakes.shp')

#land_file_10m = os.path.join(phys10m_dir, 'ne_10m_land.shp')
land_file_10m = os.path.join(cult10m_dir, 'ne_10m_admin_0_sovereignty.shp')
land_boundaries_file_10m = os.path.join(phys10m_dir, 'ne_10m_land.shp')
boundaries_file_10m = os.path.join(cult10m_dir, 'ne_10m_admin_0_boundary_lines_land.shp')
countries_file_10m = os.path.join(cult10m_dir, 'ne_10m_admin_0_countries.shp')
lakes_file_10m = os.path.join(phys10m_dir, 'ne_10m_lakes.shp')

disputed_file = os.path.join(cult50m_dir, 'ne_50m_admin_0_breakaway_disputed_areas.shp')
tiny_file = os.path.join(cult50m_dir, 'ne_50m_admin_0_tiny_countries.shp')

lakes_size = 3

def report_error(msg):
    sys.stderr.write("**ERROR**: {0}\n".format(msg))

# Command line arguments

parser = argparse.ArgumentParser(description="Creates a map from a data file")

size_group = parser.add_mutually_exclusive_group()
size_group.add_argument('--size', nargs=2, metavar=('W', 'H'),
                        type=int, default=None,
                        help="the size of output images")
size_group.add_argument('--xd', action='store_true',
                        help="xd size (should be set in the input data file)")
size_group.add_argument('--hd', action='store_true',
                        help="0.5 * (xd size)")
size_group.add_argument('--sd', action='store_true',
                        help="0.25 * (xd size)")

data_group = parser.add_mutually_exclusive_group()
data_group.add_argument('--50m', dest='use50m', action='store_true',
                        help="use 50m data")
data_group.add_argument('--10m', dest='use10m', action='store_true',
                        help="use 10m data (default)")

parser.add_argument('--png8', action='store_true',
                    help="8-bit PNG images")

parser.add_argument('--out', metavar='DIR',
                    help="the output directory")

parser.add_argument('--color', default='red',
                    help="polygon fill color for countries (use 'none' for no color)")

#parser.add_argument('--line-color', default='black',
#                    help="border line color (use 'none' for no borders)")

parser.add_argument('--scale', type=float, default=1.0,
                    help="scale for lines")

parser.add_argument('--no-markers', action='store_true',
                    help="do not draw markers")

parser.add_argument('--test', action='store_true',
                    help="produce one map only")

parser.add_argument('input_file',
                    help="input JSON data file")

# Parse arguments and load the input file

args = parser.parse_args()

with open(args.input_file) as f:
    data = json.load(f)

class CountryInfo:
    def __init__(self, data):
        self.disputed = None
        self.one_color = False
        self.marker_size = None
        if isinstance(data, unicode):
            self.name = data.encode()
        elif isinstance(data, str):
            self.name = data
        else:
            assert(isinstance(data, dict))
            self.name = data['name'].encode()
            if 'disputed' in data:
                self.disputed = data['disputed'].encode()
            if 'one-color' in data:
                self.one_color = data['one-color']
            if 'marker-size' in data:
                self.marker_size = tuple(data['marker-size'])

countries = []
for country in data['countries']:
    countries.append(CountryInfo(country))

# Validate data

if 'xd-size' not in data:
    report_error("'xd-size' is not defined in {0}".format(args.input_file))
    exit(2)

if ('proj' not in data) or ('bbox' not in data):
    report_error("'proj' or 'bbox' are not defined in {0}".format(args.input_file))
    exit(2)

xd_width, xd_height = data['xd-size']

if args.use50m:
    land_file = land_file_50m
    land_boundaries_file = land_boundaries_file_50m
    boundaries_file = boundaries_file_50m
    countries_file = countries_file_50m
    lakes_file = lakes_file_50m
else:
    land_file = land_file_10m
    land_boundaries_file = land_boundaries_file_10m
    boundaries_file = boundaries_file_10m
    countries_file = countries_file_10m
    lakes_file = lakes_file_10m

# Validate arguments

if args.sd:
    width, height = int(xd_width * 0.25), int(xd_height * 0.25)
    args.scale *= 0.25
elif args.hd:
    width, height = int(xd_width * 0.5), int(xd_height * 0.5)
    args.scale *= 0.5
elif args.xd:
    width, height = xd_width, xd_height
elif args.size:
    width, height = args.size
else:
    width, height = xd_width, xd_height

if args.color == 'none':
    args.color = None

#if args.line_color == 'none':
#    args.line_color = None

if args.scale < 0.01 or args.scale > 10:
    sys.stderr.write("\nBad scale: {0}\n\n".format(args.scale))
    sys.exit(1)

if width < 1 or height < 1 or width > 10000 or height > 10000:
    sys.stderr.write("\nBad image size: {0} x {1}\n\n".format(width, height))
    sys.exit(1)

if not args.out:
    args.out = "out_{0}_{1}_{2}".format(data['name'], width, height)

if not os.path.exists(args.out):
    os.makedirs(args.out)

# Styles and layers

# Land

def add_layer_with_style(m, layer, style, style_name=None):
    if not style_name:
        style_name = layer.name + 'Style'
    m.append_style(style_name, style)
    layer.styles.append(style_name)
    m.layers.append(layer)

def land_style():
    s = Style()
    r = Rule()

    ps = PolygonSymbolizer()
    ps.fill = Color('#f1daa0')
    r.symbols.append(ps)

#    stk = Stroke(Color('#4fadc2'), 5.0)
#    ls = LineSymbolizer(stk)
#    r.symbols.append(ls)

    s.rules.append(r)
    return s

def land_layer():
    ds = Shapefile(file=land_file)
    layer = Layer('Land')
    layer.datasource = ds
    return layer

# Land boundaries

def land_boundaries_style():
    s = Style()
    r = Rule()

    stk = Stroke(Color('#4fadc2'), 1.0)
    ls = LineSymbolizer(stk)
    r.symbols.append(ls)

#    ps = PolygonSymbolizer()
#    ps.fill = Color('red')
#    r.symbols.append(ps)

    s.rules.append(r)
    return s

def land_boundaries_layer():
    ds = Shapefile(file=land_boundaries_file)
    layer = Layer('Land Boundaries')
    layer.datasource = ds
    return layer

# Lakes

def lakes_style():
    s = Style()
    r = Rule()

    r.filter = Expression("[scalerank] <= {0}".format(lakes_size))

    ps = PolygonSymbolizer()
    ps.fill = Color('#b3e2ee')
    r.symbols.append(ps)

    stk = Stroke(Color('#4fadc2'), 1.0)
    ls = LineSymbolizer(stk)
    r.symbols.append(ls)

    s.rules.append(r)
    return s

def lakes_layer():
    ds = Shapefile(file=lakes_file)
    layer = Layer('Lakes')
    layer.datasource = ds
    return layer

# Boundaries of countries

def boundaries_style():
    s = Style()
    r = Rule()

    stk = Stroke()
#    stk.add_dash(8, 4)
#    stk.add_dash(2, 2)
#    stk.add_dash(2, 2)
    stk.color = Color('#808080')
    stk.width = 1.0
    ls = LineSymbolizer(stk)
    r.symbols.append(ls)

    s.rules.append(r)
    return s

def boundaries_layer():
    ds = Shapefile(file=boundaries_file)
    layer = Layer('Boundaries')
    layer.datasource = ds
    return layer

# A country

def country_style(name):
    s = Style()
    r = Rule()

    r.filter = Expression("[admin] = '{0}'".format(name))

    if args.color:
        ps = PolygonSymbolizer()
        ps.fill = Color(args.color)
        r.symbols.append(ps)

    s.rules.append(r)
    return s

def country_layer(name):
    ds = Shapefile(file=countries_file)
    layer = Layer("Country " + name)
    layer.datasource = ds
    return layer

def disputed_style(name, one_color=False):
    s = Style()
    r = Rule()

    r.filter = Expression("[name] = '{0}'".format(name))

    ps = PolygonSymbolizer()
    if one_color and args.color:
        ps.fill = Color(args.color)
    else:
        ps.fill = Color('#f76d50')
    r.symbols.append(ps)

    s.rules.append(r)
    return s

def disputed_layer(name):
    ds = Shapefile(file=disputed_file)
    layer = Layer("Disputed " + name)
    layer.datasource = ds
    return layer

def tiny_style(name, size):
    s = Style()
    r = Rule()

    r.filter = Expression("[name] = '{0}' or [sovereignt] = '{0}'".format(name))

    ms = MarkersSymbolizer()
    ms.fill = Color('red')
    ms.opacity = 0.5
    ms.stroke = Stroke(Color('black'), 0.0)
    ms.width = Expression('{0}'.format(size[0]))
    ms.height = Expression('{0}'.format(size[1]))
    r.symbols.append(ms)

    s.rules.append(r)
    return s

def tiny_layer(name):
    ds = Shapefile(file=tiny_file)
    layer = Layer("Tiny " + name)
    layer.datasource = ds
    return layer

# Base map

def base_map(data, width, height):
    m = Map(width, height, data['proj'].encode())
    m.background = Color('#b3e2ee')
    add_layer_with_style(m, land_layer(),
                         land_style(), 'Land Style')
    add_layer_with_style(m, boundaries_layer(),
                         boundaries_style(), 'Boundaries Style')
    add_layer_with_style(m, lakes_layer(),
                         lakes_style(), 'Lakes Style')
    add_layer_with_style(m, land_boundaries_layer(),
                         land_boundaries_style(), 'Land Boundaries Style')
    m.zoom_to_box(Box2d(*data['bbox']))
    return m

# A map with a country

def set_country(m, info):
    style = country_style(info.name)
    layer = country_layer(info.name)
    style_name = 'Style ' + info.name
    m.append_style(style_name, style)
    layer.styles.append(style_name)

    while len(m.layers) > 4:
        del m.layers[1]

    m.layers[1:1] = layer

    if info.disputed:
        style = disputed_style(info.disputed, one_color=info.one_color)
        layer = disputed_layer(info.disputed)
        style_name = 'Disputed Style ' + info.disputed
        m.append_style(style_name, style)
        layer.styles.append(style_name)
        m.layers[2:2] = layer

    if info.marker_size and not args.no_markers:
        style = tiny_style(info.name, info.marker_size)
        layer = tiny_layer(info.name)
        style_name = 'Tiny Style ' + info.name
        m.append_style(style_name, style)
        layer.styles.append(style_name)
        m.layers[2:2] = layer

# The main script

from mapnik import *

out_format = 'png256' if args.png8 else 'png'

m = base_map(data, width, height)
render_to_file(m, os.path.join(args.out, 'a.png'), out_format, args.scale)

if args.test:
    print("done (test)")
    exit(0)

for country in countries:
    print("Processing: {0}".format(country.name))
#    if country.name != 'Mauritius':
#        continue
    set_country(m, country)
    out_name = os.path.join(args.out, "{0}.png".format(country.name))
    render_to_file(m, out_name, out_format, args.scale)

print("done")
<|endoftext|>


<commit_before>#!/usr/bin/python
"""Build juju for windows and darwin on 386 and amd64."""

from __future__ import print_function

from argparse import ArgumentParser
import os
import shutil
import subprocess
import sys
import traceback

GOLANG_VERSION = '1.2.1'
CROSSCOMPILE_SOURCE = (
    'https://raw.githubusercontent.com'
    '/davecheney/golang-crosscompile/master/crosscompile.bash')
INNO_SOURCE = 'http://www.jrsoftware.org/download.php/is-unicode.exe?site=1'

def run_command(command, dry_run=False, verbose=False):
    """Optionally xecute a command and maybe print the output."""
    if verbose:
        print('Executing: %s' % command)
    if not dry_run:
        output = subprocess.check_output(command)
        if verbose:
            print(output)

def setup_cross_building(build_dir, dry_run=False, verbose=False):
    """Setup a cross building directory.

    This is not implemented but this was manually done following these steps:

    mkdir crossbuild
    cd crossbuild
    sudo apt-get install dpkg-dev wine xvfb
    apt-get source golang-go={GOLANG_VERSION}*
    export GOROOT=/var/lib/jenkins/crossbuild/golang-{GOLANG_VERSION}

    wget {CROSSCOMPILE_SOURCE} -O crosscompile.bash
    source crosscompile.bash
    go-crosscompile-build darwin/amd64
    go-crosscompile-build windows/386
    go-crosscompile-build windows/amd64

    wget {INNO_SOURCE} -O isetup-5.5.5-unicode.exe
    xvfb-run wine isetup-5.5.5-unicode.exe /verysilent
    """
    print(setup_cross_building.__doc__.format(
        GOLANG_VERSION=GOLANG_VERSION, CROSSCOMPILE_SOURCE=CROSSCOMPILE_SOURCE,
        INNO_SOURCE=INNO_SOURCE))

def build_win_client(tarball_path, build_dir, dry_run=False, verbose=False):
    pass

def build_win_agent(tarball_path, build_dir, dry_run=False, verbose=False):
    pass

def build_osx_client(tarball_path, build_dir, dry_run=False, verbose=False):
    pass

def parse_args(args=None):
    """Return the argument parser for this program."""
    parser = ArgumentParser(
        "Build juju for windows and darwin on 386 and amd64.")
    parser.add_argument(
        '-d', '--dry-run', action='store_true', default=False,
        help='Do not make changes.')
    parser.add_argument(
        '-v', '--verbose', action='store_true', default=False,
        help='Increase verbosity.')
    subparsers = parser.add_subparsers(help='sub-command help', dest="command")
    # ./crossbuild setup
    parser_setup = subparsers.add_parser(
        'setup', help='Setup a cross-compiling environment')
    parser_setup.add_argument(
        '-b', '--build-dir', default='$HOME/crossbuild',
        help='The path cross build dir.')
    # ./crossbuild win-client juju-core-1.2.3.tar.gz
    parser_win_client = subparsers.add_parser(
        'win-client',
        help='Build a 386 windown juju client and an installer.')
    parser_win_client.add_argument(
        '-b', '--build-dir', default='$HOME/crossbuild',
        help='The path cross build dir.')
    parser_win_client.add_argument(
        'tarball_path', help='The path to the juju source tarball.')
    # ./crossbuild win-agent juju-core-1.2.3.tar.gz
    parser_win_agent = subparsers.add_parser(
        'win-agent', help='Build an amd64 windows juju agent.')
    parser_win_agent.add_argument(
        '-b', '--build-dir', default='$HOME/crossbuild',
        help='The path cross build dir.')
    parser_win_agent.add_argument(
        'tarball_path', help='The path to the juju source tarball.')
    # ./crossbuild osx-client juju-core-1.2.3.tar.gz
    parser_osx_client = subparsers.add_parser(
        'osx-client', help='Build an amd64 OS X client and plugins.')
    parser_osx_client.add_argument(
        '-b', '--build-dir', default='$HOME/crossbuild',
        help='The path cross build dir.')
    parser_osx_client.add_argument(
        'tarball_path', help='The path to the juju source tarball.')
    return parser.parse_args(args)

def main(argv):
    """Cross build juju for an OS, arch, and client or server."""
    args = parse_args(argv)
    try:
        if args.command == 'setup':
            setup_cross_building(
                args.build_dir, dry_run=args.dry_run, verbose=args.verbose)
        elif args.command == 'win-client':
            build_win_client(
                args.tarball_path, args.build_dir,
                dry_run=args.dry_run, verbose=args.verbose)
        elif args.command == 'win-agent':
            build_win_agent(
                args.tarball_path, args.build_dir,
                dry_run=args.dry_run, verbose=args.verbose)
        elif args.command == 'osx-client':
            build_osx_client(
                args.tarball_path, args.build_dir,
                dry_run=args.dry_run, verbose=args.verbose)
    except Exception as e:
        print(e)
        if args.verbose:
            traceback.print_tb(sys.exc_info()[2])
        return 2
    if args.verbose:
        print("Done.")
    return 0

if __name__ == '__main__':
    sys.exit(main(sys.argv[1:]))
<commit_msg>The existing winbuildtest always pipes stderr.<commit_after>#!/usr/bin/python
"""Build juju for windows and darwin on 386 and amd64."""

from __future__ import print_function

from argparse import ArgumentParser
import os
import shutil
import subprocess
import sys
import traceback

GOLANG_VERSION = '1.2.1'
CROSSCOMPILE_SOURCE = (
    'https://raw.githubusercontent.com'
    '/davecheney/golang-crosscompile/master/crosscompile.bash')
INNO_SOURCE = 'http://www.jrsoftware.org/download.php/is-unicode.exe?site=1'

def run_command(command, dry_run=False, verbose=False, **kwargs):
    """Optionally xecute a command and maybe print the output."""
    if verbose:
        print('Executing: %s' % command)
    if not dry_run:
        kwargs['stderr'] = subprocess.STDOUT
        output = subprocess.check_output(command)
        if verbose:
            print(output)

def setup_cross_building(build_dir, dry_run=False, verbose=False):
    """Setup a cross building directory.

    This is not implemented but this was manually done following these steps:

    mkdir crossbuild
    cd crossbuild
    sudo apt-get install dpkg-dev wine xvfb
    apt-get source golang-go={GOLANG_VERSION}*
    export GOROOT=/var/lib/jenkins/crossbuild/golang-{GOLANG_VERSION}

    wget {CROSSCOMPILE_SOURCE} -O crosscompile.bash
    source crosscompile.bash
    go-crosscompile-build darwin/amd64
    go-crosscompile-build windows/386
    go-crosscompile-build windows/amd64

    wget {INNO_SOURCE} -O isetup-5.5.5-unicode.exe
    xvfb-run wine isetup-5.5.5-unicode.exe /verysilent
    """
    print(setup_cross_building.__doc__.format(
        GOLANG_VERSION=GOLANG_VERSION, CROSSCOMPILE_SOURCE=CROSSCOMPILE_SOURCE,
        INNO_SOURCE=INNO_SOURCE))

def build_win_client(tarball_path, build_dir, dry_run=False, verbose=False):
    pass

def build_win_agent(tarball_path, build_dir, dry_run=False, verbose=False):
    pass

def build_osx_client(tarball_path, build_dir, dry_run=False, verbose=False):
    pass

def parse_args(args=None):
    """Return the argument parser for this program."""
    parser = ArgumentParser(
        "Build juju for windows and darwin on 386 and amd64.")
    parser.add_argument(
        '-d', '--dry-run', action='store_true', default=False,
        help='Do not make changes.')
    parser.add_argument(
        '-v', '--verbose', action='store_true', default=False,
        help='Increase verbosity.')
    subparsers = parser.add_subparsers(help='sub-command help', dest="command")
    # ./crossbuild setup
    parser_setup = subparsers.add_parser(
        'setup', help='Setup a cross-compiling environment')
    parser_setup.add_argument(
        '-b', '--build-dir', default='$HOME/crossbuild',
        help='The path cross build dir.')
    # ./crossbuild win-client juju-core-1.2.3.tar.gz
    parser_win_client = subparsers.add_parser(
        'win-client',
        help='Build a 386 windown juju client and an installer.')
    parser_win_client.add_argument(
        '-b', '--build-dir', default='$HOME/crossbuild',
        help='The path cross build dir.')
    parser_win_client.add_argument(
        'tarball_path', help='The path to the juju source tarball.')
    # ./crossbuild win-agent juju-core-1.2.3.tar.gz
    parser_win_agent = subparsers.add_parser(
        'win-agent', help='Build an amd64 windows juju agent.')
    parser_win_agent.add_argument(
        '-b', '--build-dir', default='$HOME/crossbuild',
        help='The path cross build dir.')
    parser_win_agent.add_argument(
        'tarball_path', help='The path to the juju source tarball.')
    # ./crossbuild osx-client juju-core-1.2.3.tar.gz
    parser_osx_client = subparsers.add_parser(
        'osx-client', help='Build an amd64 OS X client and plugins.')
    parser_osx_client.add_argument(
        '-b', '--build-dir', default='$HOME/crossbuild',
        help='The path cross build dir.')
    parser_osx_client.add_argument(
        'tarball_path', help='The path to the juju source tarball.')
    return parser.parse_args(args)

def main(argv):
    """Cross build juju for an OS, arch, and client or server."""
    args = parse_args(argv)
    try:
        if args.command == 'setup':
            setup_cross_building(
                args.build_dir, dry_run=args.dry_run, verbose=args.verbose)
        elif args.command == 'win-client':
            build_win_client(
                args.tarball_path, args.build_dir,
                dry_run=args.dry_run, verbose=args.verbose)
        elif args.command == 'win-agent':
            build_win_agent(
                args.tarball_path, args.build_dir,
                dry_run=args.dry_run, verbose=args.verbose)
        elif args.command == 'osx-client':
            build_osx_client(
                args.tarball_path, args.build_dir,
                dry_run=args.dry_run, verbose=args.verbose)
    except Exception as e:
        print(e)
        if args.verbose:
            traceback.print_tb(sys.exc_info()[2])
        return 2
    if args.verbose:
        print("Done.")
    return 0

if __name__ == '__main__':
    sys.exit(main(sys.argv[1:]))
<|endoftext|>


<commit_before>import asyncio
import inspect
missing = object()

def noop(*a, **kw):
    pass

async def anoop(*a, **kw):
    pass

class RestartableTask():
    def __init__(self, loop):
        '''
        task        - a coroutine which takes a single argument, an event which
                      should be set when the main task is ready to be shut down
        on_shutdown - a coroutine which takes a single argument, an event which
                      should be set when the main task is ready to be shut down

        The shutdown_complete event MUST be set by either the task coro or
        the on_shutdown coro to finish shutting down.
        '''
        self._loop = loop
        self.running = False
        self._start_shutdown = asyncio.Event(loop=self._loop)
        self._shutdown_complete = asyncio.Event(loop=self._loop)

    async def start(self):
        if self.running:
            return

        if self._start_shutdown.is_set():
            if not self._shutdown_complete.is_set():
                await self._shutdown_complete.wait()
            self._start_shutdown.clear()
            self._shutdown_complete.clear()

        self.running = True
        self._loop.create_task(self._task())

    async def stop(self):
        if not self.running:
            return
        self.running = False

        self._start_shutdown.set()
        await self._on_shutdown()
        await self._shutdown_complete.wait()

    async def _task(self):
        pass

    async def _on_shutdown(self):
        pass

class Dispatch(RestartableTask):
    ''' Dispatch unpacked **kwargs to callbacks when events occur '''
    def __init__(self, loop):
        super().__init__(loop=loop)
        self._handlers = {}
        self._queue = asyncio.Queue(loop=self._loop)
        self._resume_processing = asyncio.Event(loop=self._loop)

    def on(self, event):
        '''
        Returns a wrapper for the given event.

        Usage:

            @dispatch.on("my_event")
            def handle_my_event(foo, bar, baz):
                ...

        '''
        handler = self._handlers.get(event, None)
        if not handler:
            raise ValueError("Unknown event '{}'".format(event))
        return handler.register

    def register(self, event, params):
        '''
        Register a new event with available params.
        Raises ValueError when the event has already been registered.

        Usage:

            dispatch.register("my_event", ["foo", "bar", "baz"])

        '''
        handler = self._handlers.get(event, missing)
        if handler is not missing:
            raise ValueError("Event {} already registered".format(event))
        self._handlers[event] = EventHandler(event, params, self._loop)

    def unregister(self, event):
        '''
        Remove all registered handlers for an event.
        Silent return when event was not registered.

        Usage:

            dispatch.unregister("my_event")
            dispatch.unregister("my_event")  # no-op

        '''
        self._handlers.pop(event, None)

    def trigger(self, event, params):
        ''' Non-blocking enqueue of an event '''
        self._queue.put_nowait((event, params))
        self._resume_processing.set()

    @property
    def events(self):
        ''' Number of events currently enqueued '''
        return self._queue.qsize()

    def clear(self):
        '''
        Clear any enqueued events.

        Throws a RuntimeException if called while the Dispatcher is running
        '''
        if self.running:
            raise RuntimeError("Can't clear the queue while running")
        while self.events:
            self._queue.get_nowait()

    async def _task(self):
        ''' Main queue processor '''

        for handler in self._handlers.values():
            await handler.start()

        while self.running:
            if self.events:
                event, params = await self._queue.get()
                handler = self._handlers.get(event, noop)
                handler(params)
            else:
                # Resume on either the next `trigger` call or a `stop`
                await self._resume_processing.wait()
                self._resume_processing.clear()

        # Let the shutdown process continue
        self._shutdown_complete.set()

    async def _on_shutdown(self):
        # If the processor is waiting, resume so it can exit cleanly
        self._resume_processing.set()

        # Give all the handlers a chance to complete their pending tasks
        tasks = [handler.stop() for handler in self._handlers.values()]
        if tasks:
            await asyncio.wait(tasks, loop=self._loop)

class EventHandler(RestartableTask):
    def __init__(self, event, params, loop):
        super().__init__(loop=loop)
        self._event = event
        self._params = params
        self._callbacks = []
        self._tasks = {}

    def __call__(self, params):
        # Don't handle the call if we're shutting down
        if not self.running:
            raise RuntimeError(
                "EventHandler must be running to delegate events")

        for callback in self._callbacks:
            task = self._loop.create_task(callback(params))
            self._tasks[id(task)] = task
            task.add_done_callback(self._delegate_done)

    def _delegate_done(self, task):
        '''
        When a callback is complete, remove it from the active task set.

        Don't throw if the task has already been removed
        '''
        self._tasks.pop(id(task), None)

    def register(self, callback):
        self._validate(callback)
        wrapped = self._wrap(callback)
        self._callbacks.append(wrapped)
        return callback

    async def _on_shutdown(self):
        # Give all active tasks a chance to complete
        active_tasks = list(self._tasks.values())
        if active_tasks:
            await asyncio.wait(active_tasks, loop=self._loop)

        self._shutdown_complete.set()

    def _wrap(self, callback):
        return partial_bind(callback)

    def _validate(self, callback):
        validate_func(self._event, callback, self._params)

    def __repr__(self):
        return "EventHandler({})".format(self._event)

def validate_func(event, callback, params):
    sig = inspect.signature(callback)
    expected = set(sig.parameters)
    for param in sig.parameters.values():
        kind = param.kind
        if kind == inspect.Parameter.VAR_POSITIONAL:
            raise ValueError(
                ("function '{}' expects parameter {} to be VAR_POSITIONAL, "
                 "when it will always be a single value.  This parameter "
                 "must be either POSITIONAL_ONLY, POSITIONAL_OR_KEYWORD, or "
                 "KEYWORD_ONLY (or omitted)").format(callback.__name__,
                                                     param.name))
        if kind == inspect.Parameter.VAR_KEYWORD:
            # **kwargs are ok, as long as the **name doesn't
            # mask an actual param that the event emits.
            if param.name in params:
                # masking :(
                raise ValueError(
                    ("function '{}' expects parameter {} to be VAR_KEYWORD, "
                     "which masks an actual parameter for event {}.  This "
                     "event has the following parameters, which must not be "
                     "used as the **VAR_KEYWORD argument.  They may be "
                     "omitted").format(
                        callback.__name__, param.name, event, params))
            else:
                # Pop from expected, this will gobble up any unused params
                expected.remove(param.name)

    available = set(params)
    unavailable = expected - available
    if unavailable:
        raise ValueError(
            ("function '{}' expects the following parameters for event {} "
             "that are not available: {}.  Available parameters for this "
             "event are: {}").format(callback.__name__, event,
                                     unavailable, available))

def partial_bind(callback):
    sig = inspect.signature(callback)
    # Wrap non-coroutines so we can always `await callback(**kw)`
    if not inspect.iscoroutinefunction(callback):
        callback = asyncio.coroutine(callback)
    base = {}
    for key, param in sig.parameters.items():
        default = param.default
        #  Param has no default - use equivalent of empty
        if default is inspect.Parameter.empty:
            base[key] = None
        else:
            base[key] = default

    async def wrapper(params):
        unbound = base.copy()
        # Only map params this callback expects
        for key in base:
            new_value = params.get(key, missing)
            if new_value is not missing:
                unbound[key] = new_value
        bound = sig.bind(**unbound)
        return await callback(*bound.args, **bound.kwargs)

    return wrapper
<commit_msg>Update RestartableTask docstring<commit_after>import asyncio
import inspect
missing = object()

def noop(*a, **kw):
    pass

class RestartableTask():
    def __init__(self, loop):
        '''
        Abstract task that ensures a previous run's shutdown logic has
        completed before the next start call is allowed to continue.

        `_task` OR `_on_shutdown` MUST set self._shutdown_complete when they
        have finished cleaning up.
        '''
        self.loop = loop
        self.running = False
        self._start_shutdown = asyncio.Event(loop=self.loop)
        self._shutdown_complete = asyncio.Event(loop=self.loop)

    async def start(self):
        if self.running:
            return

        if self._start_shutdown.is_set():
            if not self._shutdown_complete.is_set():
                await self._shutdown_complete.wait()
            self._start_shutdown.clear()
            self._shutdown_complete.clear()

        self.running = True
        self.loop.create_task(self._task())

    async def stop(self):
        if not self.running:
            return
        self.running = False

        self._start_shutdown.set()
        await self._on_shutdown()
        await self._shutdown_complete.wait()

    async def _task(self):
        pass

    async def _on_shutdown(self):
        pass

class Dispatch(RestartableTask):
    ''' Dispatch unpacked **kwargs to callbacks when events occur '''
    def __init__(self, loop):
        super().__init__(loop=loop)
        self._handlers = {}
        self._queue = asyncio.Queue(loop=self.loop)
        self._resume_processing = asyncio.Event(loop=self.loop)

    def on(self, event):
        '''
        Returns a wrapper for the given event.

        Usage:

            @dispatch.on("my_event")
            def handle_my_event(foo, bar, baz):
                ...

        '''
        handler = self._handlers.get(event, None)
        if not handler:
            raise ValueError("Unknown event '{}'".format(event))
        return handler.register

    def register(self, event, params):
        '''
        Register a new event with available params.
        Raises ValueError when the event has already been registered.

        Usage:

            dispatch.register("my_event", ["foo", "bar", "baz"])

        '''
        handler = self._handlers.get(event, missing)
        if handler is not missing:
            raise ValueError("Event {} already registered".format(event))
        self._handlers[event] = EventHandler(event, params, self.loop)

    def unregister(self, event):
        '''
        Remove all registered handlers for an event.
        Silent return when event was not registered.

        Usage:

            dispatch.unregister("my_event")
            dispatch.unregister("my_event")  # no-op

        '''
        self._handlers.pop(event, None)

    def trigger(self, event, params):
        ''' Non-blocking enqueue of an event '''
        self._queue.put_nowait((event, params))
        self._resume_processing.set()

    @property
    def events(self):
        ''' Number of events currently enqueued '''
        return self._queue.qsize()

    def clear(self):
        '''
        Clear any enqueued events.

        Throws a RuntimeException if called while the Dispatcher is running
        '''
        if self.running:
            raise RuntimeError("Can't clear the queue while running")
        while self.events:
            self._queue.get_nowait()

    async def _task(self):
        ''' Main queue processor '''

        for handler in self._handlers.values():
            await handler.start()

        while self.running:
            if self.events:
                event, params = await self._queue.get()
                handler = self._handlers.get(event, noop)
                handler(params)
            else:
                # Resume on either the next `trigger` call or a `stop`
                await self._resume_processing.wait()
                self._resume_processing.clear()

        # Let the shutdown process continue
        self._shutdown_complete.set()

    async def _on_shutdown(self):
        # If the processor is waiting, resume so it can exit cleanly
        self._resume_processing.set()

        # Give all the handlers a chance to complete their pending tasks
        tasks = [handler.stop() for handler in self._handlers.values()]
        if tasks:
            await asyncio.wait(tasks, loop=self.loop)

class EventHandler(RestartableTask):
    def __init__(self, event, params, loop):
        super().__init__(loop=loop)
        self._event = event
        self._params = params
        self._callbacks = []
        self._tasks = {}

    def __call__(self, params):
        # Don't handle the call if we're shutting down
        if not self.running:
            raise RuntimeError(
                "EventHandler must be running to delegate events")

        for callback in self._callbacks:
            task = self.loop.create_task(callback(params))
            self._tasks[id(task)] = task
            task.add_done_callback(self._delegate_done)

    def _delegate_done(self, task):
        '''
        When a callback is complete, remove it from the active task set.

        Don't throw if the task has already been removed
        '''
        self._tasks.pop(id(task), None)

    def register(self, callback):
        self._validate(callback)
        wrapped = self._wrap(callback)
        self._callbacks.append(wrapped)
        return callback

    async def _on_shutdown(self):
        # Give all active tasks a chance to complete
        active_tasks = list(self._tasks.values())
        if active_tasks:
            await asyncio.wait(active_tasks, loop=self.loop)

        self._shutdown_complete.set()

    def _wrap(self, callback):
        return partial_bind(callback)

    def _validate(self, callback):
        validate_func(self._event, callback, self._params)

    def __repr__(self):
        return "EventHandler({})".format(self._event)

def validate_func(event, callback, params):
    sig = inspect.signature(callback)
    expected = set(sig.parameters)
    for param in sig.parameters.values():
        kind = param.kind
        if kind == inspect.Parameter.VAR_POSITIONAL:
            raise ValueError(
                ("function '{}' expects parameter {} to be VAR_POSITIONAL, "
                 "when it will always be a single value.  This parameter "
                 "must be either POSITIONAL_ONLY, POSITIONAL_OR_KEYWORD, or "
                 "KEYWORD_ONLY (or omitted)").format(callback.__name__,
                                                     param.name))
        if kind == inspect.Parameter.VAR_KEYWORD:
            # **kwargs are ok, as long as the **name doesn't
            # mask an actual param that the event emits.
            if param.name in params:
                # masking :(
                raise ValueError(
                    ("function '{}' expects parameter {} to be VAR_KEYWORD, "
                     "which masks an actual parameter for event {}.  This "
                     "event has the following parameters, which must not be "
                     "used as the **VAR_KEYWORD argument.  They may be "
                     "omitted").format(
                        callback.__name__, param.name, event, params))
            else:
                # Pop from expected, this will gobble up any unused params
                expected.remove(param.name)

    available = set(params)
    unavailable = expected - available
    if unavailable:
        raise ValueError(
            ("function '{}' expects the following parameters for event {} "
             "that are not available: {}.  Available parameters for this "
             "event are: {}").format(callback.__name__, event,
                                     unavailable, available))

def partial_bind(callback):
    sig = inspect.signature(callback)
    # Wrap non-coroutines so we can always `await callback(**kw)`
    if not inspect.iscoroutinefunction(callback):
        callback = asyncio.coroutine(callback)
    base = {}
    for key, param in sig.parameters.items():
        default = param.default
        #  Param has no default - use equivalent of empty
        if default is inspect.Parameter.empty:
            base[key] = None
        else:
            base[key] = default

    async def wrapper(params):
        unbound = base.copy()
        # Only map params this callback expects
        for key in base:
            new_value = params.get(key, missing)
            if new_value is not missing:
                unbound[key] = new_value
        bound = sig.bind(**unbound)
        return await callback(*bound.args, **bound.kwargs)

    return wrapper
<|endoftext|>


<commit_before>from django.conf.urls import url
from onadata.apps.fsforms.viewsets import AssignedXFormListApi, FSXFormSubmissionApi
from .views import (
        LibraryFormsListView,
        FormsListView,
        GroupListView,
        GroupCreateView,
        GroupUpdateView,
        StageListView,
        StageUpdateView,
        StageCreateView,
        add_sub_stage,
        stage_details,
        stage_add_form,
        ScheduleListView,
        ScheduleCreateView,
        ScheduleUpdateView,
        assign, fill_form_type,
        fill_details_stage,
        fill_details_schedule,
        schedule_add_form,
        AssignedFormsListView,
        html_export, instance, show, api, download_jsonform, delete_data, data_view)

# router = MultiLookupRouter(trailing_slash=False)
urlpatterns = [
        url(r'^$', LibraryFormsListView.as_view(), name='library-forms-list'),
        url(r'^assigned/$', FormsListView.as_view(), name='forms-list'),
        # assigned form list to a user(site supervisor)
        url(r'^assigned-form-list/$', AssignedFormsListView.as_view(), name='assigned-form-list'),

        url(r'^group/$', GroupListView.as_view(), name='group-list'),
        url(r'^group/add/$', GroupCreateView.as_view(), name='group-add'),
        url(r'^group/(?P<pk>[0-9]+)/$', GroupUpdateView.as_view(), name='group-edit'),

        url(r'^stage/$', StageListView.as_view(), name='stage-list'),
        url(r'^stage/add/$', StageCreateView.as_view(), name='stage-add'),
        url(r'^stage/(?P<pk>[0-9]+)/$', StageUpdateView.as_view(), name='stage-edit'),
        url(r'^stage-add-sub-stage/(?P<pk>[0-9]+)/$', add_sub_stage, name='stage-add-sub-stage'),
        url(r'^stage-detail/(?P<pk>[0-9]+)/$', stage_details, name='stage-detail'),
        url(r'^stage-add-form/(?P<pk>[0-9]+)/$', stage_add_form, name='stage-add-form'),

        url(r'^schedule/$', ScheduleListView.as_view(), name='schedule-list'),
        url(r'^schedule/add/$', ScheduleCreateView.as_view(), name='schedule-add'),
        url(r'^schedule/(?P<pk>[0-9]+)/$', ScheduleUpdateView.as_view(), name='schedule-edit'),
        url(r'^schedule-add-form/(?P<pk>[0-9]+)/$', schedule_add_form, name='schedule-add-form'),

        url(r'^assign/(?P<pk>[0-9]+)/$', assign, name='assign'),
        url(r'^fill-form-type/(?P<pk>[0-9]+)/$', fill_form_type, name='fill_form_type'),
        url(r'^fill-details-stage/(?P<pk>[0-9]+)/$', fill_details_stage, name='fill_details_stage'),
        url(r'^fill-details-schedule/(?P<pk>[0-9]+)/$', fill_details_schedule, name='fill_details_schedule'),
]

urlpatterns = urlpatterns + [
        url(r'^assignedFormList/(?P<site_id>\[0-9]+)$',
                AssignedXFormListApi.as_view({'get': 'list'}), name='form-list'),
        # url(r'^(?P<pk>[\d+^/]+)/form\.xml$',
        #         AssignedXFormListApi.as_view({'get': 'retrieve'}),
        #         name='download_xform'),
        url(r'^(?P<pk>\[0-9]+)/form\.xml$',
                'onadata.apps.fsforms.views.download_xform', name='download_xform'),

        url(r'^(?P<pk>\[0-9]+)/(?P<site_id>\d+)$', AssignedXFormListApi.as_view({'get': 'manifest'}), name='manifest-url'),

        url(r'^submission/(?P<pk>\[0-9]+)/(?P<site_id>\d+)$',
            FSXFormSubmissionApi.as_view({'post': 'create', 'head': 'create'}),
                                                        name='submissions'),
]

urlpatterns = urlpatterns + [
        url(r'reports/(?P<fsxf_id>[0-9]+)$', html_export, name='formpack_html_export'),
        url(r'^forms/(?P<fsxf_id>[0-9]+)/instance', instance, name='instance'),
]

urlpatterns = urlpatterns + [
    # kobo main urls

    url(r'^mongo_view_api/(?P<fsxf_id>[0-9]+)/api$', api, name='mongo_view_api'),
    #  kobo main view
    url(r'^show/(?P<fsxf_id>[0-9]+)$', show, name='show'),
    url(r'^forms/(?P<fsxf_id>[0-9]+)/delete_data$', delete_data, name='delete_data'),
#
]

urlpatterns = urlpatterns + [
            # kobo main urls logger vies

    url(r'^forms/(?P<fsxf_id>[0-9]+)/form\.json',  download_jsonform,  name='download_jsonform'),

]

urlpatterns = urlpatterns + [
            # kobo main urls viewer vies

    url(r'^data-view/(?P<fsxf_id>[0-9]+)$',  data_view,  name='data_view'),

]

<commit_msg>urls clean up to allow only digits<commit_after>from django.conf.urls import url
from onadata.apps.fsforms.viewsets import AssignedXFormListApi, FSXFormSubmissionApi
from .views import (
        LibraryFormsListView,
        FormsListView,
        GroupListView,
        GroupCreateView,
        GroupUpdateView,
        StageListView,
        StageUpdateView,
        StageCreateView,
        add_sub_stage,
        stage_details,
        stage_add_form,
        ScheduleListView,
        ScheduleCreateView,
        ScheduleUpdateView,
        assign, fill_form_type,
        fill_details_stage,
        fill_details_schedule,
        schedule_add_form,
        AssignedFormsListView,
        html_export, instance, show, api, download_jsonform, delete_data, data_view)

# router = MultiLookupRouter(trailing_slash=False)
urlpatterns = [
        url(r'^$', LibraryFormsListView.as_view(), name='library-forms-list'),
        url(r'^assigned/$', FormsListView.as_view(), name='forms-list'),
        # assigned form list to a user(site supervisor)
        url(r'^assigned-form-list/$', AssignedFormsListView.as_view(), name='assigned-form-list'),

        url(r'^group/$', GroupListView.as_view(), name='group-list'),
        url(r'^group/add/$', GroupCreateView.as_view(), name='group-add'),
        url(r'^group/(?P<pk>\d+)/$', GroupUpdateView.as_view(), name='group-edit'),

        url(r'^stage/$', StageListView.as_view(), name='stage-list'),
        url(r'^stage/add/$', StageCreateView.as_view(), name='stage-add'),
        url(r'^stage/(?P<pk>\d+)/$', StageUpdateView.as_view(), name='stage-edit'),
        url(r'^stage-add-sub-stage/(?P<pk>\d+)/$', add_sub_stage, name='stage-add-sub-stage'),
        url(r'^stage-detail/(?P<pk>\d+)/$', stage_details, name='stage-detail'),
        url(r'^stage-add-form/(?P<pk>\d+)/$', stage_add_form, name='stage-add-form'),

        url(r'^schedule/$', ScheduleListView.as_view(), name='schedule-list'),
        url(r'^schedule/add/$', ScheduleCreateView.as_view(), name='schedule-add'),
        url(r'^schedule/(?P<pk>\d+)/$', ScheduleUpdateView.as_view(), name='schedule-edit'),
        url(r'^schedule-add-form/(?P<pk>\d+)/$', schedule_add_form, name='schedule-add-form'),

        url(r'^assign/(?P<pk>\d+)/$', assign, name='assign'),
        url(r'^fill-form-type/(?P<pk>\d+)/$', fill_form_type, name='fill_form_type'),
        url(r'^fill-details-stage/(?P<pk>\d+)/$', fill_details_stage, name='fill_details_stage'),
        url(r'^fill-details-schedule/(?P<pk>\d+)/$', fill_details_schedule, name='fill_details_schedule'),
]

urlpatterns = urlpatterns + [
        url(r'^assignedFormList/(?P<site_id>\d+)$',
                AssignedXFormListApi.as_view({'get': 'list'}), name='form-list'),
        # url(r'^(?P<pk>[\d+^/]+)/form\.xml$',
        #         AssignedXFormListApi.as_view({'get': 'retrieve'}),
        #         name='download_xform'),
        url(r'^(?P<pk>\d+)/form\.xml$',
                'onadata.apps.fsforms.views.download_xform', name='download_xform'),

        url(r'^(?P<pk>\d+)/(?P<site_id>\[0-9+)$', AssignedXFormListApi.as_view({'get': 'manifest'}), name='manifest-url'),

        url(r'^submission/(?P<pk>\d+)/(?P<site_id>\d+)$',
            FSXFormSubmissionApi.as_view({'post': 'create', 'head': 'create'}),
                                                        name='submissions'),
]

urlpatterns = urlpatterns + [
        url(r'reports/(?P<fsxf_id>\d+)$', html_export, name='formpack_html_export'),
        url(r'^forms/(?P<fsxf_id>\d+)/instance', instance, name='instance'),
]

urlpatterns = urlpatterns + [
    # kobo main urls

    url(r'^mongo_view_api/(?P<fsxf_id>\d+)/api$', api, name='mongo_view_api'),
    #  kobo main view
    url(r'^show/(?P<fsxf_id>\d+)$', show, name='show'),
    url(r'^forms/(?P<fsxf_id>\d+)/delete_data$', delete_data, name='delete_data'),
#
]

urlpatterns = urlpatterns + [
            # kobo main urls logger vies

    url(r'^forms/(?P<fsxf_id>\d+)/form\.json',  download_jsonform,  name='download_jsonform'),

]

urlpatterns = urlpatterns + [
            # kobo main urls viewer vies

    url(r'^data-view/(?P<fsxf_id>\d+)$',  data_view,  name='data_view'),

]

<|endoftext|>


<commit_before>### ISSUE 242 TEMPORARY FIX ###
# See https://github.com/kobotoolbox/kobocat/issues/242

from celery import shared_task
from django.core.management import call_command

@shared_task(soft_time_limit=600, time_limit=900)
def fix_root_node_names(**kwargs):
    call_command(
        'fix_root_node_names',
        **kwargs
    )

###### END ISSUE 242 FIX ######

import csv
import datetime
import pytz
import zipfile
from io import BytesIO
from django.contrib.auth.models import User
from django.core.files.storage import get_storage_class
from .models import Instance, XForm

@shared_task
def generate_stats_zip(output_filename):
    REPORTS = {
        'instances.csv': {
            'model': Instance,
            'date_field': 'date_created'
        },
        'xforms.csv': {
            'model': XForm,
            'date_field': 'date_created'
        },
        'users.csv': {
            'model': User,
            'date_field': 'date_joined'
        }
    }

    def first_day_of_next_month(any_date):
        return datetime.date(
            year=any_date.year if any_date.month < 12 else any_date.year + 1,
            month=any_date.month + 1 if any_date.month < 12 else 1,
            day=1
        )

    def first_day_of_previous_month(any_date):
        return datetime.date(
            year=any_date.year if any_date.month > 1 else any_date.year - 1,
            month=any_date.month - 1 if any_date.month > 1 else 12,
            day=1
        )

    def utc_midnight(any_date):
        return datetime.datetime(
            year=any_date.year,
            month=any_date.month,
            day=any_date.day,
            tzinfo=pytz.UTC
        )

    def list_created_by_month(model, date_field):
        today = datetime.date.today()
        # Just start at January 1 of the previous year. Going back to the
        # oldest object would be great, but it's too slow right now. Django
        # 1.10 will provide a more efficient way:
        # https://docs.djangoproject.com/en/1.10/ref/models/database-functions/#trunc
        first_date = datetime.date(
            year=today.year - 1,
            month=1,
            day=1
        )
        # We *ASSUME* that primary keys increase cronologically!
        last_object = model.objects.order_by('pk').last()
        last_date = first_day_of_next_month(getattr(last_object, date_field))
        year_month_count = []
        while last_date > first_date:
            this_start_date = first_day_of_previous_month(last_date)
            this_end_date = last_date
            criteria = {
                '{}__gte'.format(date_field): utc_midnight(this_start_date),
                '{}__lt'.format(date_field): utc_midnight(this_end_date)
            }
            objects_this_month = model.objects.filter(**criteria).count()
            year_month_count.append((
                this_start_date.year,
                this_start_date.month,
                objects_this_month
            ))
            last_date = this_start_date
        return year_month_count

    default_storage = get_storage_class()()

    with default_storage.open(output_filename, 'wb') as output_file:
        zip_file = zipfile.ZipFile(output_file, 'w', zipfile.ZIP_DEFLATED)

        for filename, report_settings in REPORTS.iteritems():
            model_name_plural = report_settings[
                'model']._meta.verbose_name_plural
            fieldnames = [
                'Year',
                'Month',
                'New {}'.format(model_name_plural.capitalize()),
                'NOTE: Records created prior to January 1 of last '
                'year are NOT included in this report!'
            ]
            data = list_created_by_month(
                report_settings['model'], report_settings['date_field'])
            csv_io = BytesIO()
            writer = csv.DictWriter(csv_io, fieldnames=fieldnames)
            writer.writeheader()
            for row in data:
                writer.writerow(dict(zip(fieldnames, row)))
            zip_file.writestr(filename, csv_io.getvalue())
            csv_io.close()

        zip_file.close()
<commit_msg>Optimize superuser_stats by running fewer queries<commit_after>### ISSUE 242 TEMPORARY FIX ###
# See https://github.com/kobotoolbox/kobocat/issues/242

from celery import shared_task
from django.core.management import call_command

@shared_task(soft_time_limit=600, time_limit=900)
def fix_root_node_names(**kwargs):
    call_command(
        'fix_root_node_names',
        **kwargs
    )

###### END ISSUE 242 FIX ######

import csv
import datetime
import pytz
import zipfile
from io import BytesIO
from collections import defaultdict
from django.contrib.auth.models import User
from django.core.files.storage import get_storage_class
from .models import Instance, XForm

@shared_task
def generate_stats_zip(output_filename):
    REPORTS = {
        'instances.csv': {
            'model': Instance,
            'date_field': 'date_created'
        },
        'xforms.csv': {
            'model': XForm,
            'date_field': 'date_created'
        },
        'users.csv': {
            'model': User,
            'date_field': 'date_joined'
        }
    }

    def list_created_by_month(model, date_field):
        # Make a single, huge query to the database
        data_dump = list(model.objects.values_list('pk', date_field))
        # Sort by date
        data_dump = sorted(data_dump, key=lambda x: x[1])

        year_month_counts = defaultdict(lambda: defaultdict(lambda: 0))
        last_pks = defaultdict(lambda: defaultdict(lambda: 0))
        for pk, date in data_dump:
            year_month_counts[date.year][date.month] += 1
            last_pks[date.year][date.month] = pk

        results = []
        cumulative = 0
        for year in sorted(year_month_counts.keys()):
            for month in sorted(year_month_counts[year].keys()):
                cumulative += year_month_counts[year][month]
                results.append((
                    year, month, year_month_counts[year][month],
                    cumulative, last_pks[year][month]
                ))

        return results

    default_storage = get_storage_class()()
    with default_storage.open(output_filename, 'wb') as output_file:
        zip_file = zipfile.ZipFile(output_file, 'w', zipfile.ZIP_DEFLATED)
        for filename, report_settings in REPORTS.iteritems():
            model_name_plural = report_settings[
                'model']._meta.verbose_name_plural
            fieldnames = [
                'Year',
                'Month',
                'New {}'.format(model_name_plural.capitalize()),
                'Cumulative {}'.format(model_name_plural.capitalize()),
                'Last Primary Key (possible clue about deleted objects)',
            ]
            data = list_created_by_month(
                report_settings['model'], report_settings['date_field'])
            csv_io = BytesIO()
            writer = csv.DictWriter(csv_io, fieldnames=fieldnames)
            writer.writeheader()
            for row in data:
                writer.writerow(dict(zip(fieldnames, row)))
            zip_file.writestr(filename, csv_io.getvalue())
            csv_io.close()

        zip_file.close()
<|endoftext|>


<commit_before>from django.conf.urls import url
from . import views

# app_name = 'nomi'

urlpatterns = [
    # nominations/
    url(r'^$', views.index, name='index'),
    url(r'^admin_portal$', views.admin_portal, name='admin_portal'),

    # nominations/clubs
    url(r'^clubs', views.club_list, name='club_list'),

    # nominations/2
    url(r'^apply/(?P<pk>\d+)/$', views.nomi_apply, name='nomi_apply'),

    # nominations/profile/
    url(r'^profile/$', views.profile_view, name='profile'),

    # nominations/profile/create
    url(r'profile/create/$', views.UserProfileCreate.as_view(), name='profile_create'),

    # nominations/profile/update/2
    url(r'^profile/update/(?P<pk>\d+)/$', views.UserProfileUpdate.as_view(), name='profile_update'),

    # nominations/result/2
    url(r'^result/(?P<pk>\d+)/$', views.result, name='result'),

    # nominations/applicants/2
    url(r'^applicants/(?P<pk>\d+)/$', views.applications, name='applicants'),

    # nominations/answers/2
    url(r'^answers/(?P<pk>\d+)/$', views.nomination_answer, name='nomi_answer'),

    # nominations/applicants/2 (redirect)
    url(r'^accept/(?P<pk>\d+)/$', views.accept_nomination, name='accept'),
    url(r'^reject/(?P<pk>\d+)/$', views.reject_nomination, name='reject'),
    url(r'^interviewed/(?P<pk>\d+)$', views.mark_as_interviewed, name='interviewed'),
    url(r'^append/fdybdjhhvjdk5878fkjjgj521/(?P<pk>\d+)$',views.append_user, name='append_user'),
    url(r'^append/fdybdjhhvjdk5878fkjjgj521/(?P<pk>\d+)$',views.replace_user, name='replace_user'),

    # nominations/create
    url(r'^create/(?P<pk>\d+)/$', views.nomination_create, name='nomi_create'),

    # nominations/2/update
    url(r'^(?P<pk>\d+)/update/$', views.NominationUpdate.as_view(), name='nomi_update'),

    # nominations/2/delete
    url(r'^(?P<pk>\d+)/delete/$', views.NominationDelete.as_view(), name='nomi_delete'),

    # nominations/post/2
    url(r'^post/(?P<pk>\d+)/$', views.post_view, name='post_view'),

    # nominations/createpost/2
    url(r'^createpost/(?P<pk>\d+)/$', views.post_create, name='post_create'),

    # nominations/club/2
    url(r'^club/(?P<pk>\d+)/$', views.club_view, name='club_view'),

    # nominations/createclub/2
    url(r'^createclub/(?P<pk>\d+)/$', views.club_create, name='club_create'),

    # nominations/club_approve/2/34
    url(r'^club_approve/(?P<club_pk>\d+)/(?P<view_pk>\d+)/$', views.club_approval, name='club_approval'),

    # nominations/post_approve/2/34
    url(r'^post_approve/(?P<post_pk>\d+)/(?P<view_pk>\d+)/$', views.post_approval, name='post_approval'),

    # nominations/final_post_approve/2/43
    url(r'^final_post_approve/(?P<post_pk>\d+)/(?P<view_pk>\d+)/$', views.final_post_approval,
        name='final_post_approval'),

    # nominations/child_post/2/8
    url(r'^child_post/(?P<pk>\d+)/$', views.child_post_view, name='child_post'),

    # nominations/child_post/2/8
    url(r'^child_club/(?P<pk>\d+)/(?P<view_pk>\d+)/$', views.child_club_view, name='child_club'),

    # nominations/nomi_detail/2/34/6
    url(r'^nomi_detail/(?P<nomi_pk>\d+)/$', views.nomi_detail, name='nomi_detail'),

    # nominations/post_approve/2/34
    url(r'^nomi_approve/(?P<nomi_pk>\d+)/$', views.nomi_approval, name='nomi_approval'),

    # nominations/final_nomi_approve/2/43/21
    url(r'^final_nomi_approve/(?P<nomi_pk>\d+)/$', views.final_nomi_approval, name='final_nomi_approval'),
    url(r'^group_nomi/(?P<pk>\d+)/$', views.group_nominations, name='nomi_group'),

]
<commit_msg>added url for result approval<commit_after>from django.conf.urls import url
from . import views

# app_name = 'nomi'

urlpatterns = [
    # nominations/
    url(r'^$', views.index, name='index'),
    url(r'^admin_portal$', views.admin_portal, name='admin_portal'),

    # nominations/clubs
    url(r'^clubs', views.club_list, name='club_list'),

    # nominations/2
    url(r'^apply/(?P<pk>\d+)/$', views.nomi_apply, name='nomi_apply'),

    # nominations/profile/
    url(r'^profile/$', views.profile_view, name='profile'),

    # nominations/profile/create
    url(r'profile/create/$', views.UserProfileCreate.as_view(), name='profile_create'),

    # nominations/profile/update/2
    url(r'^profile/update/(?P<pk>\d+)/$', views.UserProfileUpdate.as_view(), name='profile_update'),

    # nominations/result/2
    url(r'^result/(?P<pk>\d+)/$', views.result, name='result'),

    # nominations/applicants/2
    url(r'^applicants/(?P<pk>\d+)/$', views.applications, name='applicants'),

    # nominations/answers/2
    url(r'^answers/(?P<pk>\d+)/$', views.nomination_answer, name='nomi_answer'),

    # nominations/applicants/2 (redirect)
    url(r'^accept/(?P<pk>\d+)/$', views.accept_nomination, name='accept'),
    url(r'^reject/(?P<pk>\d+)/$', views.reject_nomination, name='reject'),
    url(r'^interviewed/(?P<pk>\d+)$', views.mark_as_interviewed, name='interviewed'),
    url(r'^append/fdybdjhhvjdk5878fkjjgj521/(?P<pk>\d+)$',views.append_user, name='append_user'),
    url(r'^append/fdybdjhhvjdk5878fkjjgj521/(?P<pk>\d+)$',views.replace_user, name='replace_user'),

    # nominations/create
    url(r'^create/(?P<pk>\d+)/$', views.nomination_create, name='nomi_create'),

    # nominations/2/update
    url(r'^(?P<pk>\d+)/update/$', views.NominationUpdate.as_view(), name='nomi_update'),

    # nominations/2/delete
    url(r'^(?P<pk>\d+)/delete/$', views.NominationDelete.as_view(), name='nomi_delete'),

    # nominations/post/2
    url(r'^post/(?P<pk>\d+)/$', views.post_view, name='post_view'),

    # nominations/createpost/2
    url(r'^createpost/(?P<pk>\d+)/$', views.post_create, name='post_create'),

    # nominations/club/2
    url(r'^club/(?P<pk>\d+)/$', views.club_view, name='club_view'),

    # nominations/createclub/2
    url(r'^createclub/(?P<pk>\d+)/$', views.club_create, name='club_create'),

    # nominations/club_approve/2/34
    url(r'^club_approve/(?P<club_pk>\d+)/(?P<view_pk>\d+)/$', views.club_approval, name='club_approval'),

    # nominations/post_approve/2/34
    url(r'^post_approve/(?P<post_pk>\d+)/(?P<view_pk>\d+)/$', views.post_approval, name='post_approval'),

    # nominations/final_post_approve/2/43
    url(r'^final_post_approve/(?P<post_pk>\d+)/(?P<view_pk>\d+)/$', views.final_post_approval,
        name='final_post_approval'),

    # nominations/child_post/2/8
    url(r'^child_post/(?P<pk>\d+)/$', views.child_post_view, name='child_post'),

    # nominations/child_post/2/8
    url(r'^child_club/(?P<pk>\d+)/(?P<view_pk>\d+)/$', views.child_club_view, name='child_club'),

    # nominations/nomi_detail/2/34/6
    url(r'^nomi_detail/(?P<nomi_pk>\d+)/$', views.nomi_detail, name='nomi_detail'),

    # nominations/post_approve/2/34
    url(r'^nomi_approve/(?P<nomi_pk>\d+)/$', views.nomi_approval, name='nomi_approval'),

    # nominations/final_nomi_approve/2/43/21
    url(r'^final_nomi_approve/(?P<nomi_pk>\d+)/$', views.final_nomi_approval, name='final_nomi_approval'),
    url(r'^group_nomi/(?P<pk>\d+)/$', views.group_nominations, name='nomi_group'),
    url(r'^result_approval/(?P<nomi_pk>\d+)/$', views.result_approval, name='result_approval'),

]<|endoftext|>


<commit_before># Orca
#
# Copyright 2011. Orca Team.
# Author: Joanmarie Diggs <joanmarie.diggs@gmail.com>
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2.1 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the
# Free Software Foundation, Inc., Franklin Street, Fifth Floor,
# Boston MA  02110-1301 USA.

__id__        = "$Id$"
__version__   = "$Revision$"
__date__      = "$Date$"
__copyright__ = "Copyright (c) 2011. Orca Team."
__license__   = "LGPL"

import debug
import orca
import orca_state

from scripts import apps, toolkits

_settingsManager = getattr(orca, '_settingsManager')
_eventManager = getattr(orca, '_eventManager')

class ScriptManager:

    def __init__(self):
        self.scripts = {}
        self._appModules = apps.__all__
        self._toolkitModules = toolkits.__all__
        self._defaultScript = None
        self._scriptPackages = \
            ["orca-scripts",
             "scripts",
             "scripts.apps",
             "scripts.toolkits"]
        self._appNames = \
            {'Bon Echo':         'Mozilla',
             'Deer Park':        'Mozilla',
             'Firefox':          'Mozilla',
             'Minefield':        'Mozilla',
             'Namoroka':         'Mozilla',
             'Shiretoko':        'Mozilla',
             'Lanikai':          'Thunderbird',
             'Mail/News':        'Thunderbird',
             'Miramar':          'Thunderbird',
             'Shredder':         'Thunderbird',
             'bug-buddy':        'gnome_segv2',
             'epiphany-browser': 'epiphany',
             'gaim':             'pidgin',
             'gnome-calculator': 'gcalctool',
             'gnome-help':       'yelp',
             'Nereid':           'Banshee',
             'vte':              'gnome-terminal'}

        self.setActiveScript(None, "__init__")

    def activate(self):
        """Called when this script manager is activated."""

        self._defaultScript  = None
        self.setActiveScript(self.getScript(None), "activate")

    def deactivate(self):
        """Called when this script manager is deactivated."""

        self._defaultScript  = None
        self.setActiveScript(None, "deactivate")
        self.scripts = {}

    def getModuleName(self, app):
        """Returns the module name of the script to use for application app."""

        if not (app and app.name):
            return None

        # Many python apps have an accessible name which ends in '.py'.
        # Sometimes OOo has 'soffice.bin' as its name.
        name = app.name.split('.')[0]
        altNames = self._appNames.keys()

        names = filter(lambda n: n.lower() == name.lower(), altNames)
        if names:
            name = self._appNames.get(names[0])
        else:
            for nameList in (self._appModules, self._toolkitModules):
                names = filter(lambda n: n.lower() == name.lower(), nameList)
                if names:
                    name = names[0]
                    break

        debug.println(debug.LEVEL_FINEST, "mapped %s to %s" % (app.name, name))

        return name

    def _toolkitForObject(self, obj):
        """Returns the name of the toolkit associated with obj."""

        name = ''
        if obj:
            try:
                attributes = obj.getAttributes()
            except LookupError:
                debug.println(debug.LEVEL_SEVERE,
                              "_toolkitForObject: %s no longer exists" % obj)
            else:
                attrs = dict([attr.split(':', 1) for attr in attributes])
                name = attrs.get('toolkit', '')

        return name

    def _newNamedScript(self, app, name):
        """Attempts to locate and load the named module. If successful, returns
        a script based on this module."""

        if not (app and name):
            return None

        script = None
        for package in self._scriptPackages:
            moduleName = '.'.join((package, name))
            debug.println(debug.LEVEL_FINE, "Looking for %s.py" % moduleName)
            try:
                module = __import__(moduleName, globals(), locals(), [''])
            except ImportError:
                debug.println(
                    debug.LEVEL_FINE, "Could not import %s.py" % moduleName)
                continue

            debug.println(debug.LEVEL_FINE, "Found %s.py" % moduleName)
            try:
                if hasattr(module, 'getScript'):
                    script = module.getScript(app)
                else:
                    script = module.Script(app)
                debug.println(debug.LEVEL_FINE, "Loaded %s.py" % moduleName)
                break
            except:
                debug.printException(debug.LEVEL_SEVERE)
                debug.println(
                    debug.LEVEL_SEVERE, "Could not load %s.py" % moduleName)

        return script

    def _createScript(self, app, obj=None):
        """For the given application, create a new script instance."""

        objToolkit = self._toolkitForObject(obj)
        script = self._newNamedScript(app, objToolkit)
        if script:
            return script

        moduleName = self.getModuleName(app)
        script = self._newNamedScript(app, moduleName)
        if script:
            return script

        if app and getattr(app, "toolkitName", None):
            script = self._newNamedScript(app, app.toolkitName)

        if not script:
            script = self.getDefaultScript(app)
            debug.println(debug.LEVEL_FINE, "Default script created")

        return script

    def getDefaultScript(self, app=None):
        if not app and self._defaultScript:
            return self._defaultScript

        import scripts.default as default
        script = default.Script(app)
        _eventManager.registerListeners(script)

        if not app:
            self._defaultScript = script

        return script

    def getScript(self, app, obj=None):
        """Get a script for an app (and make it if necessary).  This is used
        instead of a simple calls to Script's constructor.

        Arguments:
        - app: the Python app

        Returns an instance of a Script.
        """

        script = None

        objToolkit = self._toolkitForObject(obj)
        if objToolkit:
            script = self.scripts.get(objToolkit)
            if not script:
                script = self._createScript(app, obj)
                if script:
                    self.scripts[objToolkit] = script
                    _eventManager.registerListeners(script)
            if script:
                # Only defer to the toolkit script for this object if the
                # app script is based on a different toolkit.
                appScript = self.scripts.get(app, self.getDefaultScript())
                if issubclass(appScript.__class__, script.__class__):
                    script = appScript
                return script

        if not app:
            script = self.getDefaultScript()
        elif app in self.scripts:
            script = self.scripts[app]
        else:
            script = self._createScript(app, obj)
            self.scripts[app] = script
            _eventManager.registerListeners(script)

        return script

    def setActiveScript(self, newScript, reason=None):
        """Set the new active script.

        Arguments:
        - newScript: the new script to be made active.
        """

        if orca_state.activeScript == newScript:
            return

        if orca_state.activeScript:
            orca_state.activeScript.deactivate()

        orca_state.activeScript = newScript
        if not newScript:
            return

        _settingsManager.loadAppSettings(newScript)
        newScript.activate()
        debug.println(debug.LEVEL_FINE, "ACTIVE SCRIPT: %s (reason=%s)" \
                          % (newScript.name, reason))

    def reclaimScripts(self):
        """Compares the list of known scripts to the list of known apps,
        deleting any scripts as necessary.
        """

        from pyatspi import Registry

        try:
            desktop = Registry.getDesktop(0)
        except:
            debug.printException(debug.LEVEL_FINEST)
            return

        appList = self.scripts.keys()
        appList = filter(lambda a: a!= None and a not in desktop, appList)
        for app in appList:
            script = self.scripts.pop(app)
            _eventManager.deregisterListeners(script)
            del app
            del script
<commit_msg>Fix for bug #645364 - Need to check for "mapped" scripts before object toolkit scripts when creating a new script<commit_after># Orca
#
# Copyright 2011. Orca Team.
# Author: Joanmarie Diggs <joanmarie.diggs@gmail.com>
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2.1 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the
# Free Software Foundation, Inc., Franklin Street, Fifth Floor,
# Boston MA  02110-1301 USA.

__id__        = "$Id$"
__version__   = "$Revision$"
__date__      = "$Date$"
__copyright__ = "Copyright (c) 2011. Orca Team."
__license__   = "LGPL"

import debug
import orca
import orca_state

from scripts import apps, toolkits

_settingsManager = getattr(orca, '_settingsManager')
_eventManager = getattr(orca, '_eventManager')

class ScriptManager:

    def __init__(self):
        self.scripts = {}
        self._appModules = apps.__all__
        self._toolkitModules = toolkits.__all__
        self._defaultScript = None
        self._scriptPackages = \
            ["orca-scripts",
             "scripts",
             "scripts.apps",
             "scripts.toolkits"]
        self._appNames = \
            {'Bon Echo':         'Mozilla',
             'Deer Park':        'Mozilla',
             'Firefox':          'Mozilla',
             'Minefield':        'Mozilla',
             'Namoroka':         'Mozilla',
             'Shiretoko':        'Mozilla',
             'Lanikai':          'Thunderbird',
             'Mail/News':        'Thunderbird',
             'Miramar':          'Thunderbird',
             'Shredder':         'Thunderbird',
             'bug-buddy':        'gnome_segv2',
             'epiphany-browser': 'epiphany',
             'gaim':             'pidgin',
             'gnome-calculator': 'gcalctool',
             'gnome-help':       'yelp',
             'Nereid':           'Banshee',
             'vte':              'gnome-terminal'}

        self.setActiveScript(None, "__init__")

    def activate(self):
        """Called when this script manager is activated."""

        self._defaultScript  = None
        self.setActiveScript(self.getScript(None), "activate")

    def deactivate(self):
        """Called when this script manager is deactivated."""

        self._defaultScript  = None
        self.setActiveScript(None, "deactivate")
        self.scripts = {}

    def getModuleName(self, app):
        """Returns the module name of the script to use for application app."""

        if not (app and app.name):
            return None

        # Many python apps have an accessible name which ends in '.py'.
        # Sometimes OOo has 'soffice.bin' as its name.
        name = app.name.split('.')[0]
        altNames = self._appNames.keys()

        names = filter(lambda n: n.lower() == name.lower(), altNames)
        if names:
            name = self._appNames.get(names[0])
        else:
            for nameList in (self._appModules, self._toolkitModules):
                names = filter(lambda n: n.lower() == name.lower(), nameList)
                if names:
                    name = names[0]
                    break

        debug.println(debug.LEVEL_FINEST, "mapped %s to %s" % (app.name, name))

        return name

    def _toolkitForObject(self, obj):
        """Returns the name of the toolkit associated with obj."""

        name = ''
        if obj:
            try:
                attributes = obj.getAttributes()
            except LookupError:
                debug.println(debug.LEVEL_SEVERE,
                              "_toolkitForObject: %s no longer exists" % obj)
            else:
                attrs = dict([attr.split(':', 1) for attr in attributes])
                name = attrs.get('toolkit', '')

        return name

    def _newNamedScript(self, app, name):
        """Attempts to locate and load the named module. If successful, returns
        a script based on this module."""

        if not (app and name):
            return None

        script = None
        for package in self._scriptPackages:
            moduleName = '.'.join((package, name))
            debug.println(debug.LEVEL_FINE, "Looking for %s.py" % moduleName)
            try:
                module = __import__(moduleName, globals(), locals(), [''])
            except ImportError:
                debug.println(
                    debug.LEVEL_FINE, "Could not import %s.py" % moduleName)
                continue

            debug.println(debug.LEVEL_FINE, "Found %s.py" % moduleName)
            try:
                if hasattr(module, 'getScript'):
                    script = module.getScript(app)
                else:
                    script = module.Script(app)
                debug.println(debug.LEVEL_FINE, "Loaded %s.py" % moduleName)
                break
            except:
                debug.printException(debug.LEVEL_SEVERE)
                debug.println(
                    debug.LEVEL_SEVERE, "Could not load %s.py" % moduleName)

        return script

    def _createScript(self, app, obj=None):
        """For the given application, create a new script instance."""

        moduleName = self.getModuleName(app)
        script = self._newNamedScript(app, moduleName)
        if script:
            return script

        objToolkit = self._toolkitForObject(obj)
        script = self._newNamedScript(app, objToolkit)
        if script:
            return script

        if app and getattr(app, "toolkitName", None):
            script = self._newNamedScript(app, app.toolkitName)

        if not script:
            script = self.getDefaultScript(app)
            debug.println(debug.LEVEL_FINE, "Default script created")

        return script

    def getDefaultScript(self, app=None):
        if not app and self._defaultScript:
            return self._defaultScript

        import scripts.default as default
        script = default.Script(app)
        _eventManager.registerListeners(script)

        if not app:
            self._defaultScript = script

        return script

    def getScript(self, app, obj=None):
        """Get a script for an app (and make it if necessary).  This is used
        instead of a simple calls to Script's constructor.

        Arguments:
        - app: the Python app

        Returns an instance of a Script.
        """

        script = None

        objToolkit = self._toolkitForObject(obj)
        if objToolkit:
            script = self.scripts.get(objToolkit)
            if not script:
                script = self._createScript(app, obj)
                if script:
                    self.scripts[objToolkit] = script
                    _eventManager.registerListeners(script)
            if script:
                # Only defer to the toolkit script for this object if the
                # app script is based on a different toolkit.
                appScript = self.scripts.get(app, self.getDefaultScript())
                if issubclass(appScript.__class__, script.__class__):
                    script = appScript
                return script

        if not app:
            script = self.getDefaultScript()
        elif app in self.scripts:
            script = self.scripts[app]
        else:
            script = self._createScript(app, obj)
            self.scripts[app] = script
            _eventManager.registerListeners(script)

        return script

    def setActiveScript(self, newScript, reason=None):
        """Set the new active script.

        Arguments:
        - newScript: the new script to be made active.
        """

        if orca_state.activeScript == newScript:
            return

        if orca_state.activeScript:
            orca_state.activeScript.deactivate()

        orca_state.activeScript = newScript
        if not newScript:
            return

        _settingsManager.loadAppSettings(newScript)
        newScript.activate()
        debug.println(debug.LEVEL_FINE, "ACTIVE SCRIPT: %s (reason=%s)" \
                          % (newScript.name, reason))

    def reclaimScripts(self):
        """Compares the list of known scripts to the list of known apps,
        deleting any scripts as necessary.
        """

        from pyatspi import Registry

        try:
            desktop = Registry.getDesktop(0)
        except:
            debug.printException(debug.LEVEL_FINEST)
            return

        appList = self.scripts.keys()
        appList = filter(lambda a: a!= None and a not in desktop, appList)
        for app in appList:
            script = self.scripts.pop(app)
            _eventManager.deregisterListeners(script)
            del app
            del script
<|endoftext|>


<commit_before>from io import StringIO

from . import lexer

def test_lekxer():
    test = """# def:end )=
def :def_end=  (=return()     #
defend+_- end"""
    # Expected output token types
    expected = [
        lexer.Tokens.comment,
        lexer.Tokens.newline,

        lexer.Tokens.def_kwd,
        lexer.Tokens.typeof,
        lexer.Tokens.identifier,
        lexer.Tokens.equal,
        lexer.Tokens.group_start,
        lexer.Tokens.equal,
        lexer.Tokens.return_kwd,
        lexer.Tokens.group_start,
        lexer.Tokens.group_end,
        lexer.Tokens.comment,
        lexer.Tokens.newline,

        lexer.Tokens.identifier,
        lexer.Tokens.plus,
        lexer.Tokens.identifier,
        lexer.Tokens.minus,
        lexer.Tokens.end_kwd,
    ]
    with StringIO(test) as input:
        lex = lexer.Lexer(input).lex
        # Check if the lexed output equals the expected
        for output in expected:
            assert lex().type == output
<commit_msg>Fixed spelling error in tests<commit_after>from io import StringIO

from . import lexer

def test_lexer():
    test = """# def:end )=
def :def_end=  (=return()     #
defend+_- end"""
    # Expected output token types
    expected = [
        lexer.Tokens.comment,
        lexer.Tokens.newline,

        lexer.Tokens.def_kwd,
        lexer.Tokens.typeof,
        lexer.Tokens.identifier,
        lexer.Tokens.equal,
        lexer.Tokens.group_start,
        lexer.Tokens.equal,
        lexer.Tokens.return_kwd,
        lexer.Tokens.group_start,
        lexer.Tokens.group_end,
        lexer.Tokens.comment,
        lexer.Tokens.newline,

        lexer.Tokens.identifier,
        lexer.Tokens.plus,
        lexer.Tokens.identifier,
        lexer.Tokens.minus,
        lexer.Tokens.end_kwd,
    ]
    with StringIO(test) as input:
        lex = lexer.Lexer(input).lex
        # Check if the lexed output equals the expected
        for output in expected:
            assert lex().type == output
<|endoftext|>


<commit_before># Copyright (c) 2013, Frappe Technologies Pvt. Ltd. and contributors
# For license information, please see license.txt

from __future__ import unicode_literals
from six import iteritems
import frappe

field_map = {
	"Contact": [ "first_name", "last_name", "phone", "mobile_no", "email_id", "is_primary_contact" ],
	"Address": [ "address_line1", "address_line2", "city", "state", "pincode", "country", "is_primary_address" ]
}

def execute(filters=None):
	columns, data = get_columns(filters), get_data(filters)
	return columns, data

def get_columns(filters):
	return [
		"{reference_doctype}:Link/{reference_doctype}".format(reference_doctype=filters.get("reference_doctype")),
		"Address Line 1",
		"Address Line 2",
		"City",
		"State",
		"Postal Code",
		"Country",
		"Is Primary Address:Check",
		"First Name",
		"Last Name",
		"Phone",
		"Mobile No",
		"Email Id",
		"Is Primary Contact:Check"
	]

def get_data(filters):
	data = []
	reference_doctype = filters.get("reference_doctype")
	reference_name = filters.get("reference_name")

	return get_reference_addresses_and_contact(reference_doctype, reference_name)

def get_reference_addresses_and_contact(reference_doctype, reference_name):
	data = []
	filters = None
	reference_details = frappe._dict()

	if not reference_doctype:
		return []

	if reference_name:
		filters = { "name": reference_name }

	reference_list = [d[0] for d in frappe.get_list(reference_doctype, filters=filters, fields=["name"], as_list=True)]

	for d in reference_list:
		reference_details.setdefault(d, frappe._dict())
	reference_details = get_reference_details(reference_doctype, "Address", reference_list, reference_details)
	reference_details = get_reference_details(reference_doctype, "Contact", reference_list, reference_details)

	for reference_name, details in iteritems(reference_details):
		addresses = details.get("address", [])
		contacts  = details.get("contact", [])
		if not any([addresses, contacts]):
			result = [reference_name]
			result.extend(add_blank_columns_for("Contact"))
			result.extend(add_blank_columns_for("Address"))
			data.append(result)
		else:
			addresses = list(map(list, addresses))
			contacts = list(map(list, contacts))

			max_length = max(len(addresses), len(contacts))
			for idx in range(0, max_length):
				result = [reference_name]

				result.extend(addresses[idx] if idx < len(addresses) else add_blank_columns_for("Address"))
				result.extend(contacts[idx] if idx < len(contacts) else add_blank_columns_for("Contact"))

				data.append(result)

	return data

def get_reference_details(reference_doctype, doctype, reference_list, reference_details):
	filters =  [
		["Dynamic Link", "link_doctype", "=", reference_doctype],
		["Dynamic Link", "link_name", "in", reference_list]
	]
	fields = ["`tabDynamic Link`.link_name"] + field_map.get(doctype, [])

	records = frappe.get_list(doctype, filters=filters, fields=fields, as_list=True)
	temp_records = list()

	for d in records:
		temp_records.append(d[1:])

	reference_details[reference_list[0]][frappe.scrub(doctype)] = temp_records
	return reference_details

def add_blank_columns_for(doctype):
	return ["" for field in field_map.get(doctype, [])]
<commit_msg>fix(Addresses and Contacts Report): Frappe throw when doctype has no records (#7503)<commit_after># Copyright (c) 2013, Frappe Technologies Pvt. Ltd. and contributors
# For license information, please see license.txt

from __future__ import unicode_literals
from six import iteritems
import frappe
from frappe import _

field_map = {
	"Contact": [ "first_name", "last_name", "phone", "mobile_no", "email_id", "is_primary_contact" ],
	"Address": [ "address_line1", "address_line2", "city", "state", "pincode", "country", "is_primary_address" ]
}

def execute(filters=None):
	columns, data = get_columns(filters), get_data(filters)
	return columns, data

def get_columns(filters):
	return [
		"{reference_doctype}:Link/{reference_doctype}".format(reference_doctype=filters.get("reference_doctype")),
		"Address Line 1",
		"Address Line 2",
		"City",
		"State",
		"Postal Code",
		"Country",
		"Is Primary Address:Check",
		"First Name",
		"Last Name",
		"Phone",
		"Mobile No",
		"Email Id",
		"Is Primary Contact:Check"
	]

def get_data(filters):
	data = []
	reference_doctype = filters.get("reference_doctype")
	reference_name = filters.get("reference_name")

	return get_reference_addresses_and_contact(reference_doctype, reference_name)

def get_reference_addresses_and_contact(reference_doctype, reference_name):
	data = []
	filters = None
	reference_details = frappe._dict()

	if not reference_doctype:
		return []

	if reference_name:
		filters = { "name": reference_name }

	reference_list = [d[0] for d in frappe.get_list(reference_doctype, filters=filters, fields=["name"], as_list=True)]

	for d in reference_list:
		reference_details.setdefault(d, frappe._dict())
	reference_details = get_reference_details(reference_doctype, "Address", reference_list, reference_details)
	reference_details = get_reference_details(reference_doctype, "Contact", reference_list, reference_details)

	for reference_name, details in iteritems(reference_details):
		addresses = details.get("address", [])
		contacts  = details.get("contact", [])
		if not any([addresses, contacts]):
			result = [reference_name]
			result.extend(add_blank_columns_for("Contact"))
			result.extend(add_blank_columns_for("Address"))
			data.append(result)
		else:
			addresses = list(map(list, addresses))
			contacts = list(map(list, contacts))

			max_length = max(len(addresses), len(contacts))
			for idx in range(0, max_length):
				result = [reference_name]

				result.extend(addresses[idx] if idx < len(addresses) else add_blank_columns_for("Address"))
				result.extend(contacts[idx] if idx < len(contacts) else add_blank_columns_for("Contact"))

				data.append(result)

	return data

def get_reference_details(reference_doctype, doctype, reference_list, reference_details):
	filters =  [
		["Dynamic Link", "link_doctype", "=", reference_doctype],
		["Dynamic Link", "link_name", "in", reference_list]
	]
	fields = ["`tabDynamic Link`.link_name"] + field_map.get(doctype, [])

	records = frappe.get_list(doctype, filters=filters, fields=fields, as_list=True)
	temp_records = list()

	for d in records:
		temp_records.append(d[1:])

	if not reference_list:
		frappe.throw(_("No records present in {0}".format(reference_doctype)))

	reference_details[reference_list[0]][frappe.scrub(doctype)] = temp_records
	return reference_details

def add_blank_columns_for(doctype):
	return ["" for field in field_map.get(doctype, [])]
<|endoftext|>