<commit_before>import os
import json
import yaml
from wq.core import wq
import click

NEST = {
    'json': json,
    'yaml': yaml
}

def readfiles(basedir, ftype=None, fext=None):
    obj = {}
    if fext is None:
        fext = ftype

    for path, dirs, files in os.walk(basedir):
        if '.svn' in dirs:
            dirs.remove('.svn')
        o = obj
        if path == basedir:
            path = ""
        else:
            path = path[len(basedir) + 1:]
            apath = path.split(os.sep)
            for subdir in apath:
                o = o[subdir]
            path = os.sep.join(apath) + os.sep

        for filename in files:
            name, ext = os.path.splitext(filename)
            if ftype and ext != '.' + fext:
                continue

            fpath = path + name
            data = open(basedir + os.sep + fpath + ext)
            if ftype in NEST:
                try:
                    o[name] = NEST[ftype].load(data)
                except ValueError as e:
                    raise click.ClickException(
                        "Could not parse %s; %s" % (fpath + ext, e)
                    )
            else:
                o[name] = data.read()

        for name in dirs:
            o[name] = {}

    return obj

@wq.command()
@click.option(
    '--type', default='json', help="Source file type (e.g. json, yaml)"
)
@click.option('--extension', help="Source file extension (e.g. json, yml)")
@click.option('--output', default='output.json', help="Destination JSON file")
@click.option('--indent', default=4, help="JSON Indentation")
@click.option('--jsonp', help="Wrap as JSONP")
@click.argument('paths', type=click.Path(exists=True), nargs=-1)
def collectjson(**conf):
    """
    Load directory files into a JSON object.  The filenames will become the
    keys and the contents will become the values.  The most common use for this
    tool is to collect a group of Mustache templates into an AMD module, e.g.:

    wq collectjson --type html --output templates.js --jsonp define templates/

    This tool can also be used to collect existing JSON or YAML configuration
    files into a single object.  In that case, the file contents will be
    embeded as full JSON objects instead of strings.

    wq collectjson --type json --output config.json config/
    """

    if not conf['extension']:
        conf['extension'] = conf['type']
    if not conf['paths']:
        conf['paths'] = ['.']

    obj = {}
    for d in conf['paths']:
        obj.update(readfiles(d, conf['type'], conf['extension']))

    outfile = open(conf['output'], 'w')

    opts = {}
    if conf['indent']:
        opts['indent'] = conf['indent']

    if conf['jsonp']:
        txt = json.dumps(obj, **opts)
        txt = '%s(%s);' % (conf['jsonp'], txt)
        outfile.write(txt)
    else:
        json.dump(obj, outfile, **opts)

    click.echo('%s: %s objects collected from %s' % (
        conf['output'], len(obj), ', '.join(conf['paths'])
    ))

    outfile.close()
<commit_msg>ensure correct paths for collectjson<commit_after>import os
import json
import yaml
from wq.core import wq
import click

NEST = {
    'json': json,
    'yaml': yaml
}

def readfiles(basedir, ftype=None, fext=None):
    obj = {}
    if fext is None:
        fext = ftype

    for path, dirs, files in os.walk(basedir):
        if '.svn' in dirs:
            dirs.remove('.svn')
        o = obj
        if path == basedir:
            path = ""
        else:
            path = path[len(basedir) + 1:]
            apath = path.split(os.sep)
            for subdir in apath:
                o = o[subdir]
            path = os.sep.join(apath) + os.sep

        for filename in files:
            name, ext = os.path.splitext(filename)
            if ftype and ext != '.' + fext:
                continue

            fpath = path + name
            data = open(basedir + os.sep + fpath + ext)
            if ftype in NEST:
                try:
                    o[name] = NEST[ftype].load(data)
                except ValueError as e:
                    raise click.ClickException(
                        "Could not parse %s; %s" % (fpath + ext, e)
                    )
            else:
                o[name] = data.read()

        for name in dirs:
            o[name] = {}

    return obj

@wq.command()
@click.option(
    '--type', default='json', help="Source file type (e.g. json, yaml)"
)
@click.option('--extension', help="Source file extension (e.g. json, yml)")
@click.option('--output', default='output.json', help="Destination JSON file")
@click.option('--indent', default=4, help="JSON Indentation")
@click.option('--jsonp', help="Wrap as JSONP")
@click.argument('paths', type=click.Path(exists=True), nargs=-1)
@wq.pass_config
def collectjson(config, **conf):
    """
    Load directory files into a JSON object.  The filenames will become the
    keys and the contents will become the values.  The most common use for this
    tool is to collect a group of Mustache templates into an AMD module, e.g.:

    wq collectjson --type html --output templates.js --jsonp define templates/

    This tool can also be used to collect existing JSON or YAML configuration
    files into a single object.  In that case, the file contents will be
    embeded as full JSON objects instead of strings.

    wq collectjson --type json --output config.json config/
    """

    if not conf['extension']:
        conf['extension'] = conf['type']

    if not conf['paths']:
        if isinstance(config.get('collectjson', None), dict):
            # Ensure wq.yml paths are used if not passed via argument
            conf['paths'] = config['collectjson'].get('paths', None)

        if not conf['paths']:
            conf['paths'] = ['.']

    obj = {}
    for d in conf['paths']:
        obj.update(readfiles(d, conf['type'], conf['extension']))

    outfile = open(conf['output'], 'w')

    opts = {}
    if conf['indent']:
        opts['indent'] = conf['indent']

    if conf['jsonp']:
        txt = json.dumps(obj, **opts)
        txt = '%s(%s);' % (conf['jsonp'], txt)
        outfile.write(txt)
    else:
        json.dump(obj, outfile, **opts)

    click.echo('%s: %s objects collected from %s' % (
        conf['output'], len(obj), ', '.join(conf['paths'])
    ))

    outfile.close()
<|endoftext|>


<commit_before>#Python glue program to generate C options to be read from inifile

import StringIO

DEFAULT_OPTIONS_FILENAME = "default_options.ini"
C_HANDLER_SOURCE = "handler.c"
C_OPTIONS_HEADER = "options.h"

#(Parameter_name,category,Parameter_type,Default_value)

scalar_options_set_1=[
("Omega_m","cosmo",float,0.26),
("Omega_l","cosmo",float,0.74),
("w","cosmo",float,-1.0),
("n_scalar","cosmo",float,0.960),
("sigma8","cosmo",float,0.798),
("z_source","cosmo",float,1.0)
]

scalar_options=[
("Cosmological parameters",scalar_options_set_1)
]

def ini_string(value,ptype):
	if ptype==str:
		return value
	elif ptype in(float,int):
		return str(value)
	else:
		raise ValueError("Unknown parameter type")

def declaration_string(ptype):
	if ptype==str:
		return 'char*'
	elif ptype==int:
		return 'int'
	elif ptype==float:
		return 'float'
	else:
		raise ValueError("Unknown parameter type")

def declaration_match(ptype):
	if ptype==str:
		return 'strdup(value)'
	elif ptype==int:
		return 'atoi(value)'
	elif ptype==float:
		return 'atof(value)'
	else:
		raise ValueError("Unknown parameter type")

def generate_handler(options):
	S=StringIO.StringIO()
	S.write("""/* DO NOT EDIT THIS FILE - IT IS GENERATED FROM build_options.py. Edit that. */

#include <stdlib.h>
#include <string.h>
#include "options.h"

int handler(void *user,const char *section,const char *name, const char *value){

		sys_options *options = (sys_options*) user;

		#define MATCH(s,n) strcmp(section,s)==0 && strcmp(name,n)==0
""")

	S.write("""\n
		if(0){
		}""")

	for set_name,options_set in scalar_options:
		for name,set,ptype,default in options_set:
			S.write(""" else if(MATCH("%s","%s")){
				options->%s = %s;
				}"""%(set,name,name,declaration_match(ptype)))

	S.write(""" else{
		return 0;
		}
		return 1;
}""")

	S.seek(0)
	return S.read()

def generate_options_type(options):
	S=StringIO.StringIO()
	S.write("""/* DO NOT EDIT THIS FILE - IT IS GENERATED FROM build_options.py. Edit that. */

#ifndef __OPTIONS_H
#define __OPTIONS_H

typedef struct{

""")

	for set_name,options_set in scalar_options:
		for name,set,ptype,default in options_set:
			S.write("""%s %s;\n"""%(declaration_string(ptype),name))

	S.write("""
} sys_options;

int handler(void *,const char*,const char*,const char*);

#endif

""")

	S.seek(0)
	return S.read()

def generate_default_parameter_file(options):
	S=StringIO.StringIO()
	S.write("""# Default options ini file generated from build_options.py #\n""")

	for set_name,options_set in scalar_options:
		S.write("""
##########################
##########%s##############
##########################
[%s]

"""%(set_name,options_set[0][1]))
		for name,set,ptype,default in options_set:
			S.write("""%s = %s\n""" %(name,ini_string(default,ptype)))

		S.write("""\n""")

	S.seek(0)
	return S.read()

file(C_HANDLER_SOURCE,"w").write(generate_handler(scalar_options))
file(C_OPTIONS_HEADER,"w").write(generate_options_type(scalar_options))
file(DEFAULT_OPTIONS_FILENAME,"w").write(generate_default_parameter_file(scalar_options))
<commit_msg>Added layout for vector options<commit_after>#Python glue program to generate C options to be read from inifile

import StringIO

DEFAULT_OPTIONS_FILENAME = "default_options.ini"
C_HANDLER_SOURCE = "handler.c"
C_OPTIONS_HEADER = "options.h"

#(Parameter_name,category,Parameter_type,Default_value)

scalar_options_set_1=[
("Omega_m","cosmo",float,0.26),
("Omega_l","cosmo",float,0.74),
("w","cosmo",float,-1.0),
("n_scalar","cosmo",float,0.960),
("sigma8","cosmo",float,0.798),
("z_source","cosmo",float,1.0)
]

scalar_options=[
("Cosmological parameters scalar",scalar_options_set_1)
]

#(Number_of_parameters_name,array_parameter_name,num_elements,Parameter_type,Default_values) (only numbers supported!!)
vector_options_set_1=[
("Nsi8","si8",3,float,0.798),
("Nw","w",4,float,-1.0)
]

vector_options=[
("Cosmological parameters vector",vector_options_set_1)
]

def ini_string(value,ptype):
	if ptype==str:
		return value
	elif ptype in(float,int):
		return str(value)
	else:
		raise ValueError("Unknown parameter type")

def declaration_string(ptype):
	if ptype==str:
		return 'char*'
	elif ptype==int:
		return 'int'
	elif ptype==float:
		return 'float'
	else:
		raise ValueError("Unknown parameter type")

def declaration_match(ptype):
	if ptype==str:
		return 'strdup(value)'
	elif ptype==int:
		return 'atoi(value)'
	elif ptype==float:
		return 'atof(value)'
	else:
		raise ValueError("Unknown parameter type")

def generate_handler(scalar_options,vector_options):
	S=StringIO.StringIO()
	S.write("""/* DO NOT EDIT THIS FILE - IT IS GENERATED FROM build_options.py. Edit that. */

#include <stdlib.h>
#include <string.h>
#include "options.h"

int handler(void *user,const char *section,const char *name, const char *value){

		sys_options *options = (sys_options*) user;

		#define MATCH(s,n) strcmp(section,s)==0 && strcmp(name,n)==0
""")

	S.write("""\n
		if(0){
		}""")

	for set_name,options_set in scalar_options:
		for name,set,ptype,default in options_set:
			S.write(""" else if(MATCH("%s","%s")){
				options->%s = %s;
				}"""%(set,name,name,declaration_match(ptype)))

	S.write(""" else{
		return 0;
		}
		return 1;
}""")

	S.seek(0)
	return S.read()

def generate_options_type(scalar_options,vector_options):
	S=StringIO.StringIO()
	S.write("""/* DO NOT EDIT THIS FILE - IT IS GENERATED FROM build_options.py. Edit that. */

#ifndef __OPTIONS_H
#define __OPTIONS_H

typedef struct{

""")

	for set_name,options_set in scalar_options:
		for name,set,ptype,default in options_set:
			S.write("""%s %s;\n"""%(declaration_string(ptype),name))

	S.write("""
} sys_options;

int handler(void *,const char*,const char*,const char*);

#endif

""")

	S.seek(0)
	return S.read()

def generate_default_parameter_file(scalar_options,vector_options):
	S=StringIO.StringIO()
	S.write("""# Default options ini file generated from build_options.py #\n""")

	for set_name,options_set in scalar_options:
		S.write("""
##########################
##########%s##############
##########################
[%s]

"""%(set_name,options_set[0][1]))
		for name,set,ptype,default in options_set:
			S.write("""%s = %s\n""" %(name,ini_string(default,ptype)))

		S.write("""\n""")

	S.seek(0)
	return S.read()

file(C_HANDLER_SOURCE,"w").write(generate_handler(scalar_options,vector_options))
file(C_OPTIONS_HEADER,"w").write(generate_options_type(scalar_options,vector_options))
file(DEFAULT_OPTIONS_FILENAME,"w").write(generate_default_parameter_file(scalar_options,vector_options))
<|endoftext|>


<commit_before>import cmd
import logging
import config
import bukkit
import pkg

class Commands(cmd.Cmd):
  intro   = motd
  prompt  = 'bukget>'

  def __init__(self):
    self._parse_repository()

  def do_bukkit(self, s):
    '''bukkit [command] [options]
    This command houses all the functions related to the bukkit server
    itself.  This includes starting, stopping, updating, etc.  For help on
    the available commands, type "bukkit help" and "bukkit help [command]"
    '''
    cmd = s.split()
    if len(cmd) > 1:
      BukkitCommands().onecmd(s)
    else:
      BukkitCommands().cmdloop()

  def do_pkg(self, s):
    '''pkg [command] [options]
    This command houses all the functions related to plugin and library
    packages.  This includes installing, upgrading, removing, etc.  For help
    on the available commands, type "pkg help" and "pkg help [command]"
    '''
    cmd = s.split()
    if len(cmd) > 1:
      PackageCommands().onecmd(s)
    else:
      PackageCommands().cmdloop()

  def do_player(self, s):
    '''player [command] [options]
    This command houses all the functions related to player control.  This
    includes banning, giving, opping, etc.  For help on the available
    commands, type "player help" and "player help command"
    '''
    cmd = s.split()
    if len(cmd) > 1:
      PlayerCommands().onecmd(s)
    else:
      PlayerCommands().cmdloop()

  def do_snapshot(self, s):
    pass

  def do_backup(self, s):
    pass

  def do_ip(self, s):
    pass

  def do_time(self, s):
    pass

  def do_msg(self, s):
    pass

  def do_update(self, s):
    pass

class BukkitCommands(cmd.Cmd):
  prompt  = 'bukget[BUKKIT]>'
  bukkit  = bukkit.Bukkit()

  def do_info(self, s):
    '''info
    Returns the version information, branch information, and running status
    of the bukkit server.
    '''
    status  = {True: 'running', False: 'stopped'}
    print 'Current Bukkit Build %s on Branch %s is currently %s.' %\
          (self.server.version,
           self.server.branch,
           status[self.server.running()])

  def do_start(self, s):
    '''start
    Starts the bukkit server.
    '''
    if not self.server.running():
      self.server.start()
      print 'Server has started.'
    else:
      print 'Server already running!'

  def do_stop(self, s):
    '''stop
    Stops the bukkit server.
    '''
    if self.server.running():
      self.server.stop()
      print 'Server shutdown initiated.'
    else:
      print 'Server already stopped!'

  def do_console(self, s):
    '''console
    Presents the user the server console.  When the user wants to exit the
    console they must use the following key combinations: Cntrl+a then d.
    '''
    if self.server.running():
      os.system('screen -DRS bukkit_server')
    else:
      print 'Could not connect to console.'

  def do_upgrade(self, s):
    '''upgrade [branch/build number]
    Upgrades the installed bukkit version to the requested branch or build
    number.
    '''
    if not self.server.running():
      self.server.download(s)

  def do_return(self, s):
    '''return
    Returns the user back to the main command prompt.  This is mostly used
    if the user is using interactive mode.
    '''
    pass

class PlayerCommands(cmd.Cmd):
  prompt  = 'bukget[PLAYER]>'
  server  = bukkit.Bukkit()

  def do_list(self, s):
    '''list
    Returns the list of players online.
    '''
    print 'Players online: %s' % ', '.join(self.server.player_list())

  def do_msg(self, s):
    '''msg [player] [message]
    Sends the message to the specified player.
    '''
    dset      = s.split()
    if len(dset) > 1:
      msg     = ' '.join(dset[1:])
      player  = dset[0]
      self.server.player_msg(player, msg)
      print 'Message sent to %s' % player
    else:
      print 'Not enough arguments.'

  def do_give(self, s):
    '''give [player] [item-id] [amount]
    Will give the player specified the amount (default 64) of the item
    requested.  The command will first parse the item definitions to see if
    the item-id is a known item within the system.  If not then it will fall
    back on simply sending the item-id given directly to the server.  For
    example:

      player give ManiacM4c stone

    is the same as:

      player give ManiacM4c 1
    '''
    dset = s.split()
    if len(dset) < 2:
      print 'Not enough arguments.'
      return

    # Loading the item definition dictionary and parsing out the options
    idefs   = json.loads(open(os.path.join(config.get('Paths', 'config'),
                        'item_definitions.json'), 'r'))
    player  = dset[0]
    item_id = dset[1]

    # If the item id exists within the item definition, then we need to get
    # the real item id and set that instead.
    if item_id in idefs:
      item_id = idefs[item_id]

    # if there is amount specified use that, otherwise wet it to 64.
    if len(dset) <= 3:
      amount = dset[3]
    else:
      amount = 64

    # Now to send the command to the server.
    self.server.player_give(player, item_id, amount)
    print 'Server instructed to give %s %s to %s' % (amount, item_id, player)

  def do_teleport(self, s):
    '''teleport [player to teleport] [destination player]
    Teleports the player to the destination player's location.
    '''
    dset = s.split()
    if len(dset) < 2:
      print 'Not enough parameters.'
      return
    player_tp   = dset[0]
    player_dest = dset[1]
    self.server.player_teleport(player_tp, player_dest)
    print 'Server instructed to teleport %s to %s.' % (player_tp, player_dest)

  def do_op(self, s):
    '''op [player]
    Gives the player operator (op) priviledges.
    '''
    if len(s) < 3:
      print 'Not a valid user.  Must be 3 characters or greater.'
      return
    self.server.player_op(s)
    print 'Server instructed to op %s.' % s

  def do_deop(self, s):
    '''deop [player]
    Removed the player's operator (op) priviledges.
    '''
    if len(s) < 3:
      print 'Not a valid user.  Must be 3 characters or greater.'
      return
    self.server.player_deop(s)
    print 'Server instructed to deop %s.' % s

  def do_kick(self, s):
    '''kick [player]
    Kicks the player from the server.
    '''
    if len(s) < 3:
      print 'Not a valid user.  Must be 3 characters or greater.'
      return
    self.server.player_kick(s)
    print 'Server instructed to kick %s.' % s

  def do_ban(self, s):
    '''ban [player]
    Bans the player from being able to play on the server.
    '''
    if len(s) < 3:
      print 'Not a valid user.  Must be 3 characters or greater.'
      return
    self.server.player_ban(s)
    print 'Server instructed to ban %s.' % s

  def do_pardon(self, s):
    '''pardon [player]
    Removes the player ban so that theplayer can play on the server.
    '''
    if len(s) < 3:
      print 'Not a valid user.  Must be 3 characters or greater.'
      return
    self.server.player_pardon(s)
    print 'Server instructed to pardon %s.' % s<commit_msg>Started package commands class.<commit_after>import cmd
import logging
import config
import bukkit
import pkg

def prepare(self):
  '''
  This private function will check to make sure that all the pre-requisites
  to running a bukget installation are in place and will either create the
  folders as needed or error out and inform the user what needs to be done
  in order to make the installation work.  If everything was either able to
  be created or checked out ok then we should be returning True, else we will
  return False.
  '''
  pass

class Commands(cmd.Cmd):
  intro   = motd
  prompt  = 'bukget>'

  def __init__(self):
    cmd.Cmd.__init__(self)

  def do_bukkit(self, s):
    '''bukkit [command] [options]
    This command houses all the functions related to the bukkit server
    itself.  This includes starting, stopping, updating, etc.  For help on
    the available commands, type "bukkit help" and "bukkit help [command]"
    '''
    cmd = s.split()
    if len(cmd) > 1:
      BukkitCommands().onecmd(s)
    else:
      BukkitCommands().cmdloop()

  def do_pkg(self, s):
    '''pkg [command] [options]
    This command houses all the functions related to plugin and library
    packages.  This includes installing, upgrading, removing, etc.  For help
    on the available commands, type "pkg help" and "pkg help [command]"
    '''
    cmd = s.split()
    if len(cmd) > 1:
      PackageCommands().onecmd(s)
    else:
      PackageCommands().cmdloop()

  def do_player(self, s):
    '''player [command] [options]
    This command houses all the functions related to player control.  This
    includes banning, giving, opping, etc.  For help on the available
    commands, type "player help" and "player help command"
    '''
    cmd = s.split()
    if len(cmd) > 1:
      PlayerCommands().onecmd(s)
    else:
      PlayerCommands().cmdloop()

  def do_snapshot(self, s):
    pass

  def do_backup(self, s):
    pass

  def do_ip(self, s):
    pass

  def do_time(self, s):
    pass

  def do_msg(self, s):
    pass

  def do_update(self, s):
    pass

class BukkitCommands(cmd.Cmd):
  prompt  = 'bukget[BUKKIT]>'
  bukkit  = bukkit.Bukkit()

  def do_info(self, s):
    '''info
    Returns the version information, branch information, and running status
    of the bukkit server.
    '''
    status  = {True: 'running', False: 'stopped'}
    print 'Current Bukkit Build %s on Branch %s is currently %s.' %\
          (self.server.version,
           self.server.branch,
           status[self.server.running()])

  def do_start(self, s):
    '''start
    Starts the bukkit server.
    '''
    if not self.server.running():
      self.server.start()
      print 'Server has started.'
    else:
      print 'Server already running!'

  def do_stop(self, s):
    '''stop
    Stops the bukkit server.
    '''
    if self.server.running():
      self.server.stop()
      print 'Server shutdown initiated.'
    else:
      print 'Server already stopped!'

  def do_console(self, s):
    '''console
    Presents the user the server console.  When the user wants to exit the
    console they must use the following key combinations: Cntrl+a then d.
    '''
    if self.server.running():
      os.system('screen -DRS bukkit_server')
    else:
      print 'Could not connect to console.'

  def do_upgrade(self, s):
    '''upgrade [branch/build number]
    Upgrades the installed bukkit version to the requested branch or build
    number.
    '''
    if not self.server.running():
      self.server.download(s)

  def do_return(self, s):
    '''return
    Returns the user back to the main command prompt.  This is mostly used
    if the user is using interactive mode.
    '''
    pass

class PlayerCommands(cmd.Cmd):
  prompt  = 'bukget[PLAYER]>'
  server  = bukkit.Bukkit()

  def do_list(self, s):
    '''list
    Returns the list of players online.
    '''
    print 'Players online: %s' % ', '.join(self.server.player_list())

  def do_msg(self, s):
    '''msg [player] [message]
    Sends the message to the specified player.
    '''
    dset      = s.split()
    if len(dset) > 1:
      msg     = ' '.join(dset[1:])
      player  = dset[0]
      self.server.player_msg(player, msg)
      print 'Message sent to %s' % player
    else:
      print 'Not enough arguments.'

  def do_give(self, s):
    '''give [player] [item-id] [amount]
    Will give the player specified the amount (default 64) of the item
    requested.  The command will first parse the item definitions to see if
    the item-id is a known item within the system.  If not then it will fall
    back on simply sending the item-id given directly to the server.  For
    example:

      player give ManiacM4c stone

    is the same as:

      player give ManiacM4c 1
    '''
    dset = s.split()
    if len(dset) < 2:
      print 'Not enough arguments.'
      return

    # Loading the item definition dictionary and parsing out the options
    idefs   = json.loads(open(os.path.join(config.get('Paths', 'config'),
                        'item_definitions.json'), 'r'))
    player  = dset[0]
    item_id = dset[1]

    # If the item id exists within the item definition, then we need to get
    # the real item id and set that instead.
    if item_id in idefs:
      item_id = idefs[item_id]

    # if there is amount specified use that, otherwise wet it to 64.
    if len(dset) <= 3:
      amount = dset[3]
    else:
      amount = 64

    # Now to send the command to the server.
    self.server.player_give(player, item_id, amount)
    print 'Server instructed to give %s %s to %s' % (amount, item_id, player)

  def do_teleport(self, s):
    '''teleport [player to teleport] [destination player]
    Teleports the player to the destination player's location.
    '''
    dset = s.split()
    if len(dset) < 2:
      print 'Not enough parameters.'
      return
    player_tp   = dset[0]
    player_dest = dset[1]
    self.server.player_teleport(player_tp, player_dest)
    print 'Server instructed to teleport %s to %s.' % (player_tp, player_dest)

  def do_op(self, s):
    '''op [player]
    Gives the player operator (op) priviledges.
    '''
    if len(s) < 3:
      print 'Not a valid user.  Must be 3 characters or greater.'
      return
    self.server.player_op(s)
    print 'Server instructed to op %s.' % s

  def do_deop(self, s):
    '''deop [player]
    Removed the player's operator (op) priviledges.
    '''
    if len(s) < 3:
      print 'Not a valid user.  Must be 3 characters or greater.'
      return
    self.server.player_deop(s)
    print 'Server instructed to deop %s.' % s

  def do_kick(self, s):
    '''kick [player]
    Kicks the player from the server.
    '''
    if len(s) < 3:
      print 'Not a valid user.  Must be 3 characters or greater.'
      return
    self.server.player_kick(s)
    print 'Server instructed to kick %s.' % s

  def do_ban(self, s):
    '''ban [player]
    Bans the player from being able to play on the server.
    '''
    if len(s) < 3:
      print 'Not a valid user.  Must be 3 characters or greater.'
      return
    self.server.player_ban(s)
    print 'Server instructed to ban %s.' % s

  def do_pardon(self, s):
    '''pardon [player]
    Removes the player ban so that theplayer can play on the server.
    '''
    if len(s) < 3:
      print 'Not a valid user.  Must be 3 characters or greater.'
      return
    self.server.player_pardon(s)
    print 'Server instructed to pardon %s.' % s

  def do_return(self, s):
    '''return
    Returns the user to the main command prompt.
    '''

class PackageCommands(cmd.Cmd):
  prompt  = 'bukget[PKG]>'
  pkgs    = []
  local   = []

  def __init__(self):
    cmd.Cmd.__init__(self)
    self._get_pkgs()

  def _get_pkgs(self):
    pass<|endoftext|>


<commit_before># Copyright 2011 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

#!/usr/bin/env python

import datetime
import logging
import os
import sys
import uuid

from django.utils import simplejson as json
from google.appengine.ext import db
from google.appengine.ext import webapp
from google.appengine.ext.webapp.util import run_wsgi_app

os.environ['DJANGO_SETTINGS_MODULE'] = 'settings'
from django.template.loader import render_to_string

from lib.py import belay

def server_url(path):
  return belay.this_server_url_prefix() + path

# to remove
def server_feed_url(path, feed_id):
  return server_url(path + "?" + feed_id)

# to remove
def server_cap(path, feed_id):
  return { '@': server_feed_url(path, feed_id) }

class FeedData(db.Model):
  title = db.StringProperty()
  name = db.StringProperty()
  location = db.StringProperty()

class SnapshotData(db.Model):
  snapshot = db.TextProperty()

class ItemData(db.Model):
  when = db.DateTimeProperty(auto_now_add=True);
  body = db.TextProperty(required=True);
  via = db.TextProperty();

  def nicedate(self):
    date = self.when
    today = datetime.datetime.now()
    date_ordinal = date.toordinal()
    today_ordinal = today.toordinal()
    format = ""
    if date_ordinal == today_ordinal:
      format = "today"
    elif date_ordinal == today_ordinal - 1:
      format = "yesterday"
    elif date_ordinal > today_ordinal - 7:
      format = '%a'
    elif date.year == today.year:
      format = '%a, %b %d'
    else:
      format = '%a, %b %d %Y'
    format += ' - %I:%M %p'
    return date.strftime(format)

def render_to_response(handler, tmpl_filename, dictionary):
    """Note that this is different than Django's similarly named function"""
    content = render_to_string(tmpl_filename, dictionary)
    # django is misguided here - it doesn't read the file as UTF-8
    handler.xhr_content(content, "text/html;charset=UTF-8")

class BaseHandler(belay.BcapHandler):
  class InvalidFeed(Exception):
    pass

  def validate_feed(self):
    try:
      feed_uuid = uuid.UUID(self.request.query_string)
      return str(feed_uuid)
    except:
      raise BaseHandler.InvalidFeed()

  def handle_exception(self, exc, debug_mode):
    if isinstance(exc,BaseHandler.InvalidFeed):
      self.error(404)
    else:
      super(BaseHandler, self).handle_exception(exc, debug_mode)

class LaunchHandler(belay.CapHandler):
  def get(self):
    feed = self.get_entity()
    feed_id = feed.key().name()
    snapshot = SnapshotData.all().ancestor(feed).get()

    response = {
      'page': {
        'html': server_url('/buzzer-belay.html'),
        'window': { 'width': 300, 'height': 400 }
      },
      'gadget': {
        'html': belay.regrant(EditorViewHandler, feed),
        'scripts': [ server_url("/buzzer.js") ]
      },
      'info': {
        'post_cap': belay.regrant(DataPostHandler, feed),
        'snapshot_cap': belay.regrant(SnapshotHandler, snapshot),
        'snapshot': snapshot.snapshot,
        'reader_gen_cap': server_cap("/belay/generateReader", feed_id),
        'editor_cap': belay.regrant(EditorViewHandler, feed),
        'readChitURL': server_url("/chit-24.png"),
        'postChitURL': server_url("/chit-25.png")
      },
      'attributes': {
        'set': belay.regrant(SetAttributesHandler, feed)
      }
    }

    self.bcapResponse(response)

class SnapshotHandler(belay.CapHandler):
  def get(self):
    self.bcapResponse(self.get_entity().snapshot)

  def put(self):
    v = self.bcapRequest()
    snapshot = self.get_entity()
    snapshot.snapshot = v
    snapshot.put()
    self.bcapNullResponse()

class LaunchReaderHandler(BaseHandler):
  def get(self):
    feed_id = self.validate_feed();
    feed = FeedData.get_by_key_name(feed_id);

    response = {
      'page': {
        'html': server_url('/buzzer-belay.html'),
        'window': { 'width': 300, 'height': 400 }
      },
      'gadget': {
        'html': belay.regrant(ReaderViewHandler, feed),
        'scripts': [ server_url("/buzzer.js") ]
      },
      'info': {
        'editor_cap': belay.regrant(ReaderViewHandler, feed),
      }
    }

    self.bcapResponse(response)

class GenerateProfileHandler(BaseHandler):
  def post(self):
    feed_uuid = uuid.uuid4()
    feed_id = str(feed_uuid)
    feed = FeedData(key_name=feed_id);
    feed.title = self.request.get('title')
    feed.put()
    snapshot = SnapshotData(parent=feed)
    snapshot.put()

    response = {
      'launch': belay.regrant(LaunchHandler, feed),
      'icon': server_url("/tool-buzzer.png"),
      'name': feed.title
    }

    self.bcapResponse(response)

class GenerateReaderHandler(BaseHandler):
  def get(self):
    feed_id = self.validate_feed();
    feed = FeedData.get_by_key_name(feed_id);
    response = {
        'launch': server_cap('/belay/launchReader', feed_id),
        'name': 'buzz about ' + feed.title,
        'icon': server_url('/tool-buzzer.png')
    };
    self.bcapResponse(response)

class ViewHandler(belay.CapHandler):
  def get(self):
    feed = self.get_entity()
    need_profile = feed.title is None
    has_name = feed.name and feed.name != ''
    has_location = feed.location and feed.location != ''

    q = ItemData.all();
    q.ancestor(feed);
    q.order('-when');
    items = q.fetch(10);

    render_to_response(self, 'buzzer.tmpl',
      { 'css_url': server_url('/buzzer.css'),
        'chit_read_url': server_url('/chit-24.png'),
        'chit_post_url': server_url('/chit-25.png'),
        'post_url': belay.regrant(DataPostHandler, feed).serialize(),
        'profile_url': belay.regrant(DataProfileHandler, feed).serialize(),
        'include_post': self.include_post(),
        'need_profile': need_profile,
        'feed': feed,
        'items': items,
        'has_name': has_name,
        'has_location': has_location,
        'has_subtitle': has_name or has_location

      })

class EditorViewHandler(ViewHandler):
  def include_post(self):
    return True

class ReaderViewHandler(ViewHandler):
  def include_post(self):
    return False

class DataProfileHandler(belay.CapHandler):
  def post(self):
    feed = self.get_entity()

    request = self.bcapRequest()
    feed.title = request.get('title')
    feed.put()

    self.bcapNullResponse()

class DataPostHandler(belay.CapHandler):
  def post(self):
    feed = self.get_entity()

    request = self.bcapRequest()
    v = request.get('body', '')
    if not v:
      self.error(400, "Empty or missing posting body")
      return

    item = ItemData(parent=feed, body=request['body'])
    v = request.get('when', '')
    if v:
      item.when = v
    v = request.get('via', '')
    if v:
      item.via = v
    item.put()

    self.bcapNullResponse()

class SetAttributesHandler(belay.CapHandler):
  def put(self):
    feed = self.get_entity()
    request = self.bcapRequest()

    feed.name = request.get('name', '')
    feed.location = request.get('location', '')
    feed.put()

    self.bcapNullResponse()

application = webapp.WSGIApplication(
  [(r'/cap/.*', belay.ProxyHandler),
  ('/belay/launchReader', LaunchReaderHandler),
  ('/belay/generateProfile', GenerateProfileHandler),
  ('/belay/generateReader', GenerateReaderHandler),
  ],
  debug=True)

# Internal Cap Paths
belay.set_handlers(
  '/cap',
  [ ('/belay/launch', LaunchHandler),
    ('/view/editor', EditorViewHandler),
    ('/view/reader', ReaderViewHandler),
    ('/data/profile', DataProfileHandler),
    ('/data/snapshot', SnapshotHandler),
    ('/data/post', DataPostHandler),
    ('/data/attributes', SetAttributesHandler),
  ])

def main():
  logging.getLogger().setLevel(logging.DEBUG)
  run_wsgi_app(application)

if __name__ == "__main__":
  main()
<commit_msg>generate read-only launch moved to "good caps"<commit_after># Copyright 2011 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

#!/usr/bin/env python

import datetime
import logging
import os
import sys
import uuid

from django.utils import simplejson as json
from google.appengine.ext import db
from google.appengine.ext import webapp
from google.appengine.ext.webapp.util import run_wsgi_app

os.environ['DJANGO_SETTINGS_MODULE'] = 'settings'
from django.template.loader import render_to_string

from lib.py import belay

def server_url(path):
  return belay.this_server_url_prefix() + path

class FeedData(db.Model):
  title = db.StringProperty()
  name = db.StringProperty()
  location = db.StringProperty()

class SnapshotData(db.Model):
  snapshot = db.TextProperty()

class ItemData(db.Model):
  when = db.DateTimeProperty(auto_now_add=True);
  body = db.TextProperty(required=True);
  via = db.TextProperty();

  def nicedate(self):
    date = self.when
    today = datetime.datetime.now()
    date_ordinal = date.toordinal()
    today_ordinal = today.toordinal()
    format = ""
    if date_ordinal == today_ordinal:
      format = "today"
    elif date_ordinal == today_ordinal - 1:
      format = "yesterday"
    elif date_ordinal > today_ordinal - 7:
      format = '%a'
    elif date.year == today.year:
      format = '%a, %b %d'
    else:
      format = '%a, %b %d %Y'
    format += ' - %I:%M %p'
    return date.strftime(format)

def render_to_response(handler, tmpl_filename, dictionary):
    """Note that this is different than Django's similarly named function"""
    content = render_to_string(tmpl_filename, dictionary)
    # django is misguided here - it doesn't read the file as UTF-8
    handler.xhr_content(content, "text/html;charset=UTF-8")

class BaseHandler(belay.BcapHandler):
  class InvalidFeed(Exception):
    pass

  def validate_feed(self):
    try:
      feed_uuid = uuid.UUID(self.request.query_string)
      return str(feed_uuid)
    except:
      raise BaseHandler.InvalidFeed()

  def handle_exception(self, exc, debug_mode):
    if isinstance(exc,BaseHandler.InvalidFeed):
      self.error(404)
    else:
      super(BaseHandler, self).handle_exception(exc, debug_mode)

class LaunchHandler(belay.CapHandler):
  def get(self):
    feed = self.get_entity()
    snapshot = SnapshotData.all().ancestor(feed).get()

    response = {
      'page': {
        'html': server_url('/buzzer-belay.html'),
        'window': { 'width': 300, 'height': 400 }
      },
      'gadget': {
        'html': belay.regrant(EditorViewHandler, feed),
        'scripts': [ server_url("/buzzer.js") ]
      },
      'info': {
        'post_cap': belay.regrant(DataPostHandler, feed),
        'snapshot_cap': belay.regrant(SnapshotHandler, snapshot),
        'snapshot': snapshot.snapshot,
        'reader_gen_cap': belay.regrant(GenerateReaderHandler, feed),
        'editor_cap': belay.regrant(EditorViewHandler, feed),
        'readChitURL': server_url("/chit-24.png"),
        'postChitURL': server_url("/chit-25.png")
      },
      'attributes': {
        'set': belay.regrant(SetAttributesHandler, feed)
      }
    }

    self.bcapResponse(response)

class SnapshotHandler(belay.CapHandler):
  def get(self):
    self.bcapResponse(self.get_entity().snapshot)

  def put(self):
    v = self.bcapRequest()
    snapshot = self.get_entity()
    snapshot.snapshot = v
    snapshot.put()
    self.bcapNullResponse()

class LaunchReaderHandler(belay.CapHandler):
  def get(self):
    feed = self.get_entity()

    response = {
      'page': {
        'html': server_url('/buzzer-belay.html'),
        'window': { 'width': 300, 'height': 400 }
      },
      'gadget': {
        'html': belay.regrant(ReaderViewHandler, feed),
        'scripts': [ server_url("/buzzer.js") ]
      },
      'info': {
        'editor_cap': belay.regrant(ReaderViewHandler, feed),
      }
    }

    self.bcapResponse(response)

class GenerateProfileHandler(BaseHandler):
  def post(self):
    feed_uuid = uuid.uuid4()
    feed_id = str(feed_uuid)
    feed = FeedData(key_name=feed_id);
    feed.title = self.request.get('title')
    feed.put()
    snapshot = SnapshotData(parent=feed)
    snapshot.put()

    response = {
      'launch': belay.regrant(LaunchHandler, feed),
      'icon': server_url("/tool-buzzer.png"),
      'name': feed.title
    }

    self.bcapResponse(response)

class GenerateReaderHandler(belay.CapHandler):
  def get(self):
    feed = self.get_entity()
    response = {
        'launch': belay.regrant(LaunchReaderHandler, feed),
        'name': 'buzz about ' + feed.title,
        'icon': server_url('/tool-buzzer.png')
    };
    self.bcapResponse(response)

class ViewHandler(belay.CapHandler):
  def get(self):
    feed = self.get_entity()
    need_profile = feed.title is None
    has_name = feed.name and feed.name != ''
    has_location = feed.location and feed.location != ''

    q = ItemData.all();
    q.ancestor(feed);
    q.order('-when');
    items = q.fetch(10);

    render_to_response(self, 'buzzer.tmpl',
      { 'css_url': server_url('/buzzer.css'),
        'chit_read_url': server_url('/chit-24.png'),
        'chit_post_url': server_url('/chit-25.png'),
        'post_url': belay.regrant(DataPostHandler, feed).serialize(),
        'profile_url': belay.regrant(DataProfileHandler, feed).serialize(),
        'include_post': self.include_post(),
        'need_profile': need_profile,
        'feed': feed,
        'items': items,
        'has_name': has_name,
        'has_location': has_location,
        'has_subtitle': has_name or has_location

      })

class EditorViewHandler(ViewHandler):
  def include_post(self):
    return True

class ReaderViewHandler(ViewHandler):
  def include_post(self):
    return False

class DataProfileHandler(belay.CapHandler):
  def post(self):
    feed = self.get_entity()

    request = self.bcapRequest()
    feed.title = request.get('title')
    feed.put()

    self.bcapNullResponse()

class DataPostHandler(belay.CapHandler):
  def post(self):
    feed = self.get_entity()

    request = self.bcapRequest()
    v = request.get('body', '')
    if not v:
      self.error(400, "Empty or missing posting body")
      return

    item = ItemData(parent=feed, body=request['body'])
    v = request.get('when', '')
    if v:
      item.when = v
    v = request.get('via', '')
    if v:
      item.via = v
    item.put()

    self.bcapNullResponse()

class SetAttributesHandler(belay.CapHandler):
  def put(self):
    feed = self.get_entity()
    request = self.bcapRequest()

    feed.name = request.get('name', '')
    feed.location = request.get('location', '')
    feed.put()

    self.bcapNullResponse()

application = webapp.WSGIApplication(
  [(r'/cap/.*', belay.ProxyHandler),
  ('/belay/generateProfile', GenerateProfileHandler),
  ],
  debug=True)

# Internal Cap Paths
belay.set_handlers(
  '/cap',
  [ ('/belay/launch', LaunchHandler),
    ('/belay/generateReader', GenerateReaderHandler),
    ('/belay/launchReader', LaunchReaderHandler),
    ('/view/editor', EditorViewHandler),
    ('/view/reader', ReaderViewHandler),
    ('/data/profile', DataProfileHandler),
    ('/data/snapshot', SnapshotHandler),
    ('/data/post', DataPostHandler),
    ('/data/attributes', SetAttributesHandler),
  ])

def main():
  logging.getLogger().setLevel(logging.DEBUG)
  run_wsgi_app(application)

if __name__ == "__main__":
  main()
<|endoftext|>


<commit_before># coding=utf-8

__version__ = '0.4.9'
<commit_msg>Version 0.5.0<commit_after># coding=utf-8

__version__ = '0.5.0'
<|endoftext|>


<commit_before># vim: tabstop=4 shiftwidth=4 softtabstop=4

# Copyright 2010 United States Government as represented by the
# Administrator of the National Aeronautics and Space Administration.
# All Rights Reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

"""Base classes for our unit tests.

Allows overriding of flags for use of fakes, and some black magic for
inline callbacks.

"""

import eventlet
eventlet.monkey_patch(os=False)

import copy
import gettext
import os
import shutil
import sys
import tempfile
import uuid

import fixtures
from oslo.config import cfg
import testtools

from nova import context
from nova import db
from nova.db import migration
from nova.network import manager as network_manager
from nova.objects import base as objects_base
from nova.openstack.common.db.sqlalchemy import session
from nova.openstack.common.fixture import moxstubout
from nova.openstack.common import log as logging
from nova.openstack.common import timeutils
from nova import paths
from nova import service
from nova.tests import conf_fixture
from nova.tests import policy_fixture

test_opts = [
    cfg.StrOpt('sqlite_clean_db',
               default='clean.sqlite',
               help='File name of clean sqlite db'),
    ]

CONF = cfg.CONF
CONF.register_opts(test_opts)
CONF.import_opt('connection',
                'nova.openstack.common.db.sqlalchemy.session',
                group='database')
CONF.import_opt('sqlite_db', 'nova.openstack.common.db.sqlalchemy.session')
CONF.import_opt('enabled', 'nova.api.openstack', group='osapi_v3')
CONF.set_override('use_stderr', False)

logging.setup('nova')

_DB_CACHE = None

class Database(fixtures.Fixture):

    def __init__(self, db_session, db_migrate, sql_connection,
                    sqlite_db, sqlite_clean_db):
        self.sql_connection = sql_connection
        self.sqlite_db = sqlite_db
        self.sqlite_clean_db = sqlite_clean_db

        self.engine = db_session.get_engine()
        self.engine.dispose()
        conn = self.engine.connect()
        if sql_connection == "sqlite://":
            if db_migrate.db_version() > db_migrate.INIT_VERSION:
                return
        else:
            testdb = paths.state_path_rel(sqlite_db)
            if os.path.exists(testdb):
                return
        db_migrate.db_sync()
        if sql_connection == "sqlite://":
            conn = self.engine.connect()
            self._DB = "".join(line for line in conn.connection.iterdump())
            self.engine.dispose()
        else:
            cleandb = paths.state_path_rel(sqlite_clean_db)
            shutil.copyfile(testdb, cleandb)

    def setUp(self):
        super(Database, self).setUp()

        if self.sql_connection == "sqlite://":
            conn = self.engine.connect()
            conn.connection.executescript(self._DB)
            self.addCleanup(self.engine.dispose)
        else:
            shutil.copyfile(paths.state_path_rel(self.sqlite_clean_db),
                            paths.state_path_rel(self.sqlite_db))

class SampleNetworks(fixtures.Fixture):

    """Create sample networks in the database."""

    def __init__(self, host=None):
        self.host = host

    def setUp(self):
        super(SampleNetworks, self).setUp()
        ctxt = context.get_admin_context()
        network = network_manager.VlanManager(host=self.host)
        bridge_interface = CONF.flat_interface or CONF.vlan_interface
        network.create_networks(ctxt,
                                label='test',
                                cidr='10.0.0.0/8',
                                multi_host=CONF.multi_host,
                                num_networks=CONF.num_networks,
                                network_size=CONF.network_size,
                                cidr_v6=CONF.fixed_range_v6,
                                gateway=CONF.gateway,
                                gateway_v6=CONF.gateway_v6,
                                bridge=CONF.flat_network_bridge,
                                bridge_interface=bridge_interface,
                                vpn_start=CONF.vpn_start,
                                vlan_start=CONF.vlan_start,
                                dns1=CONF.flat_network_dns)
        for net in db.network_get_all(ctxt):
            network.set_network_host(ctxt, net)

class ReplaceModule(fixtures.Fixture):
    """Replace a module with a fake module."""

    def __init__(self, name, new_value):
        self.name = name
        self.new_value = new_value

    def _restore(self, old_value):
        sys.modules[self.name] = old_value

    def setUp(self):
        super(ReplaceModule, self).setUp()
        old_value = sys.modules.get(self.name)
        sys.modules[self.name] = self.new_value
        self.addCleanup(self._restore, old_value)

class ServiceFixture(fixtures.Fixture):
    """Run a service as a test fixture."""

    def __init__(self, name, host=None, **kwargs):
        name = name
        host = host and host or uuid.uuid4().hex
        kwargs.setdefault('host', host)
        kwargs.setdefault('binary', 'nova-%s' % name)
        self.kwargs = kwargs

    def setUp(self):
        super(ServiceFixture, self).setUp()
        self.service = service.Service.create(**self.kwargs)
        self.service.start()
        self.addCleanup(self.service.kill)

class TranslationFixture(fixtures.Fixture):
    """Use gettext NullTranslation objects in tests."""

    def setUp(self):
        super(TranslationFixture, self).setUp()
        nulltrans = gettext.NullTranslations()
        gettext_fixture = fixtures.MonkeyPatch('gettext.translation',
                                               lambda *x, **y: nulltrans)
        self.gettext_patcher = self.useFixture(gettext_fixture)

class TestingException(Exception):
    pass

class TestCase(testtools.TestCase):
    """Test case base class for all unit tests.

    Due to the slowness of DB access, please consider deriving from
    `NoDBTestCase` first.
    """
    USES_DB = True

    def setUp(self):
        """Run before each test method to initialize test environment."""
        super(TestCase, self).setUp()
        test_timeout = os.environ.get('OS_TEST_TIMEOUT', 0)
        try:
            test_timeout = int(test_timeout)
        except ValueError:
            # If timeout value is invalid do not set a timeout.
            test_timeout = 0
        if test_timeout > 0:
            self.useFixture(fixtures.Timeout(test_timeout, gentle=True))
        self.useFixture(fixtures.NestedTempfile())
        self.useFixture(fixtures.TempHomeDir())
        self.useFixture(TranslationFixture())

        if (os.environ.get('OS_STDOUT_CAPTURE') == 'True' or
                os.environ.get('OS_STDOUT_CAPTURE') == '1'):
            stdout = self.useFixture(fixtures.StringStream('stdout')).stream
            self.useFixture(fixtures.MonkeyPatch('sys.stdout', stdout))
        if (os.environ.get('OS_STDERR_CAPTURE') == 'True' or
                os.environ.get('OS_STDERR_CAPTURE') == '1'):
            stderr = self.useFixture(fixtures.StringStream('stderr')).stream
            self.useFixture(fixtures.MonkeyPatch('sys.stderr', stderr))

        self.log_fixture = self.useFixture(fixtures.FakeLogger())
        self.useFixture(conf_fixture.ConfFixture(CONF))

        if self.USES_DB:
            global _DB_CACHE
            if not _DB_CACHE:
                _DB_CACHE = Database(session, migration,
                        sql_connection=CONF.database.connection,
                        sqlite_db=CONF.sqlite_db,
                        sqlite_clean_db=CONF.sqlite_clean_db)

            self.useFixture(_DB_CACHE)

        # NOTE(danms): Make sure to reset us back to non-remote objects
        # for each test to avoid interactions. Also, backup the object
        # registry.
        objects_base.NovaObject.indirection_api = None
        self._base_test_obj_backup = copy.copy(
            objects_base.NovaObject._obj_classes)
        self.addCleanup(self._restore_obj_registry)

        mox_fixture = self.useFixture(moxstubout.MoxStubout())
        self.mox = mox_fixture.mox
        self.stubs = mox_fixture.stubs
        self.addCleanup(self._clear_attrs)
        self.useFixture(fixtures.EnvironmentVariable('http_proxy'))
        self.policy = self.useFixture(policy_fixture.PolicyFixture())
        CONF.set_override('fatal_exception_format_errors', True)
        CONF.set_override('enabled', True, 'osapi_v3')
        CONF.set_override('force_dhcp_release', False)
        # This will be cleaned up by the NestedTempfile fixture
        CONF.set_override('lock_path', tempfile.mkdtemp())

    def _restore_obj_registry(self):
        objects_base.NovaObject._obj_classes = self._base_test_obj_backup

    def _clear_attrs(self):
        # Delete attributes that don't start with _ so they don't pin
        # memory around unnecessarily for the duration of the test
        # suite
        for key in [k for k in self.__dict__.keys() if k[0] != '_']:
            del self.__dict__[key]

    def flags(self, **kw):
        """Override flag variables for a test."""
        group = kw.pop('group', None)
        for k, v in kw.iteritems():
            CONF.set_override(k, v, group)

    def start_service(self, name, host=None, **kwargs):
        svc = self.useFixture(ServiceFixture(name, host, **kwargs))
        return svc.service

class APICoverage(object):

    cover_api = None

    def test_api_methods(self):
        self.assertTrue(self.cover_api is not None)
        api_methods = [x for x in dir(self.cover_api)
                       if not x.startswith('_')]
        test_methods = [x[5:] for x in dir(self)
                        if x.startswith('test_')]
        self.assertThat(
            test_methods,
            testtools.matchers.ContainsAll(api_methods))

class TimeOverride(fixtures.Fixture):
    """Fixture to start and remove time override."""

    def setUp(self):
        super(TimeOverride, self).setUp()
        timeutils.set_time_override()
        self.addCleanup(timeutils.clear_time_override)

class NoDBTestCase(TestCase):
    """
    `NoDBTestCase` differs from TestCase in that DB access is not supported.
    This makes tests run significantly faster. If possible, all new tests
    should derive from this class.
    """
    USES_DB = False
<commit_msg>Include name/level in unit test log messages<commit_after># vim: tabstop=4 shiftwidth=4 softtabstop=4

# Copyright 2010 United States Government as represented by the
# Administrator of the National Aeronautics and Space Administration.
# All Rights Reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

"""Base classes for our unit tests.

Allows overriding of flags for use of fakes, and some black magic for
inline callbacks.

"""

import eventlet
eventlet.monkey_patch(os=False)

import copy
import gettext
import os
import shutil
import sys
import tempfile
import uuid

import fixtures
from oslo.config import cfg
import testtools

from nova import context
from nova import db
from nova.db import migration
from nova.network import manager as network_manager
from nova.objects import base as objects_base
from nova.openstack.common.db.sqlalchemy import session
from nova.openstack.common.fixture import moxstubout
from nova.openstack.common import log as logging
from nova.openstack.common import timeutils
from nova import paths
from nova import service
from nova.tests import conf_fixture
from nova.tests import policy_fixture

test_opts = [
    cfg.StrOpt('sqlite_clean_db',
               default='clean.sqlite',
               help='File name of clean sqlite db'),
    ]

CONF = cfg.CONF
CONF.register_opts(test_opts)
CONF.import_opt('connection',
                'nova.openstack.common.db.sqlalchemy.session',
                group='database')
CONF.import_opt('sqlite_db', 'nova.openstack.common.db.sqlalchemy.session')
CONF.import_opt('enabled', 'nova.api.openstack', group='osapi_v3')
CONF.set_override('use_stderr', False)

logging.setup('nova')

_DB_CACHE = None

class Database(fixtures.Fixture):

    def __init__(self, db_session, db_migrate, sql_connection,
                    sqlite_db, sqlite_clean_db):
        self.sql_connection = sql_connection
        self.sqlite_db = sqlite_db
        self.sqlite_clean_db = sqlite_clean_db

        self.engine = db_session.get_engine()
        self.engine.dispose()
        conn = self.engine.connect()
        if sql_connection == "sqlite://":
            if db_migrate.db_version() > db_migrate.INIT_VERSION:
                return
        else:
            testdb = paths.state_path_rel(sqlite_db)
            if os.path.exists(testdb):
                return
        db_migrate.db_sync()
        if sql_connection == "sqlite://":
            conn = self.engine.connect()
            self._DB = "".join(line for line in conn.connection.iterdump())
            self.engine.dispose()
        else:
            cleandb = paths.state_path_rel(sqlite_clean_db)
            shutil.copyfile(testdb, cleandb)

    def setUp(self):
        super(Database, self).setUp()

        if self.sql_connection == "sqlite://":
            conn = self.engine.connect()
            conn.connection.executescript(self._DB)
            self.addCleanup(self.engine.dispose)
        else:
            shutil.copyfile(paths.state_path_rel(self.sqlite_clean_db),
                            paths.state_path_rel(self.sqlite_db))

class SampleNetworks(fixtures.Fixture):

    """Create sample networks in the database."""

    def __init__(self, host=None):
        self.host = host

    def setUp(self):
        super(SampleNetworks, self).setUp()
        ctxt = context.get_admin_context()
        network = network_manager.VlanManager(host=self.host)
        bridge_interface = CONF.flat_interface or CONF.vlan_interface
        network.create_networks(ctxt,
                                label='test',
                                cidr='10.0.0.0/8',
                                multi_host=CONF.multi_host,
                                num_networks=CONF.num_networks,
                                network_size=CONF.network_size,
                                cidr_v6=CONF.fixed_range_v6,
                                gateway=CONF.gateway,
                                gateway_v6=CONF.gateway_v6,
                                bridge=CONF.flat_network_bridge,
                                bridge_interface=bridge_interface,
                                vpn_start=CONF.vpn_start,
                                vlan_start=CONF.vlan_start,
                                dns1=CONF.flat_network_dns)
        for net in db.network_get_all(ctxt):
            network.set_network_host(ctxt, net)

class ReplaceModule(fixtures.Fixture):
    """Replace a module with a fake module."""

    def __init__(self, name, new_value):
        self.name = name
        self.new_value = new_value

    def _restore(self, old_value):
        sys.modules[self.name] = old_value

    def setUp(self):
        super(ReplaceModule, self).setUp()
        old_value = sys.modules.get(self.name)
        sys.modules[self.name] = self.new_value
        self.addCleanup(self._restore, old_value)

class ServiceFixture(fixtures.Fixture):
    """Run a service as a test fixture."""

    def __init__(self, name, host=None, **kwargs):
        name = name
        host = host and host or uuid.uuid4().hex
        kwargs.setdefault('host', host)
        kwargs.setdefault('binary', 'nova-%s' % name)
        self.kwargs = kwargs

    def setUp(self):
        super(ServiceFixture, self).setUp()
        self.service = service.Service.create(**self.kwargs)
        self.service.start()
        self.addCleanup(self.service.kill)

class TranslationFixture(fixtures.Fixture):
    """Use gettext NullTranslation objects in tests."""

    def setUp(self):
        super(TranslationFixture, self).setUp()
        nulltrans = gettext.NullTranslations()
        gettext_fixture = fixtures.MonkeyPatch('gettext.translation',
                                               lambda *x, **y: nulltrans)
        self.gettext_patcher = self.useFixture(gettext_fixture)

class TestingException(Exception):
    pass

class TestCase(testtools.TestCase):
    """Test case base class for all unit tests.

    Due to the slowness of DB access, please consider deriving from
    `NoDBTestCase` first.
    """
    USES_DB = True

    def setUp(self):
        """Run before each test method to initialize test environment."""
        super(TestCase, self).setUp()
        test_timeout = os.environ.get('OS_TEST_TIMEOUT', 0)
        try:
            test_timeout = int(test_timeout)
        except ValueError:
            # If timeout value is invalid do not set a timeout.
            test_timeout = 0
        if test_timeout > 0:
            self.useFixture(fixtures.Timeout(test_timeout, gentle=True))
        self.useFixture(fixtures.NestedTempfile())
        self.useFixture(fixtures.TempHomeDir())
        self.useFixture(TranslationFixture())

        if (os.environ.get('OS_STDOUT_CAPTURE') == 'True' or
                os.environ.get('OS_STDOUT_CAPTURE') == '1'):
            stdout = self.useFixture(fixtures.StringStream('stdout')).stream
            self.useFixture(fixtures.MonkeyPatch('sys.stdout', stdout))
        if (os.environ.get('OS_STDERR_CAPTURE') == 'True' or
                os.environ.get('OS_STDERR_CAPTURE') == '1'):
            stderr = self.useFixture(fixtures.StringStream('stderr')).stream
            self.useFixture(fixtures.MonkeyPatch('sys.stderr', stderr))

        fs = '%(levelname)s [%(name)s] %(message)s'
        self.log_fixture = self.useFixture(fixtures.FakeLogger(format=fs))
        self.useFixture(conf_fixture.ConfFixture(CONF))

        if self.USES_DB:
            global _DB_CACHE
            if not _DB_CACHE:
                _DB_CACHE = Database(session, migration,
                        sql_connection=CONF.database.connection,
                        sqlite_db=CONF.sqlite_db,
                        sqlite_clean_db=CONF.sqlite_clean_db)

            self.useFixture(_DB_CACHE)

        # NOTE(danms): Make sure to reset us back to non-remote objects
        # for each test to avoid interactions. Also, backup the object
        # registry.
        objects_base.NovaObject.indirection_api = None
        self._base_test_obj_backup = copy.copy(
            objects_base.NovaObject._obj_classes)
        self.addCleanup(self._restore_obj_registry)

        mox_fixture = self.useFixture(moxstubout.MoxStubout())
        self.mox = mox_fixture.mox
        self.stubs = mox_fixture.stubs
        self.addCleanup(self._clear_attrs)
        self.useFixture(fixtures.EnvironmentVariable('http_proxy'))
        self.policy = self.useFixture(policy_fixture.PolicyFixture())
        CONF.set_override('fatal_exception_format_errors', True)
        CONF.set_override('enabled', True, 'osapi_v3')
        CONF.set_override('force_dhcp_release', False)
        # This will be cleaned up by the NestedTempfile fixture
        CONF.set_override('lock_path', tempfile.mkdtemp())

    def _restore_obj_registry(self):
        objects_base.NovaObject._obj_classes = self._base_test_obj_backup

    def _clear_attrs(self):
        # Delete attributes that don't start with _ so they don't pin
        # memory around unnecessarily for the duration of the test
        # suite
        for key in [k for k in self.__dict__.keys() if k[0] != '_']:
            del self.__dict__[key]

    def flags(self, **kw):
        """Override flag variables for a test."""
        group = kw.pop('group', None)
        for k, v in kw.iteritems():
            CONF.set_override(k, v, group)

    def start_service(self, name, host=None, **kwargs):
        svc = self.useFixture(ServiceFixture(name, host, **kwargs))
        return svc.service

class APICoverage(object):

    cover_api = None

    def test_api_methods(self):
        self.assertTrue(self.cover_api is not None)
        api_methods = [x for x in dir(self.cover_api)
                       if not x.startswith('_')]
        test_methods = [x[5:] for x in dir(self)
                        if x.startswith('test_')]
        self.assertThat(
            test_methods,
            testtools.matchers.ContainsAll(api_methods))

class TimeOverride(fixtures.Fixture):
    """Fixture to start and remove time override."""

    def setUp(self):
        super(TimeOverride, self).setUp()
        timeutils.set_time_override()
        self.addCleanup(timeutils.clear_time_override)

class NoDBTestCase(TestCase):
    """
    `NoDBTestCase` differs from TestCase in that DB access is not supported.
    This makes tests run significantly faster. If possible, all new tests
    should derive from this class.
    """
    USES_DB = False
<|endoftext|>


<commit_before>"""
Basic OCR functions.  Submit OCR tasks and retrieve the result.
"""

import re
import os
import traceback
from datetime import datetime
from celery import result as celeryresult
from celery.task.sets import TaskSet, subtask
from django.conf import settings
from django.contrib.auth.decorators import login_required
from django.db import transaction
from django.http import HttpResponse, HttpResponseRedirect, Http404
from django.shortcuts import render_to_response
from django.template import RequestContext
from django.utils import simplejson
from ocradmin.ocr import tasks
from ocradmin.ocr import utils as ocrutils
from ocradmin.ocrmodels.models import OcrModel
from ocradmin.ocrpresets.models import OcrPreset
from ocradmin.ocrtasks.models import OcrTask, OcrBatch

class AppException(StandardError):
    """
    Most generic app error.
    """
    pass

@login_required
def index(request):
    """
    OCR index page.
    """
    return HttpResponseRedirect("/ocr/convert/")

@login_required
@transaction.commit_manually
def batch(request):
    """
    Save a batch of posted image to the DFS and convert them with Celery.
    """

    # add available seg and bin presets to the context
    context = {
        "binpresets": OcrPreset.objects.filter(
            type="binarize").order_by("name"),
        "segpresets": OcrPreset.objects.filter(
            type="segment").order_by("name"),
    }
    tasktype = "Batch"
    celerytask = tasks.ConvertPageTask

    if not request.method == "POST":
        return render_to_response("ocr/batch.html", context,
                        context_instance=RequestContext(request))

    try:
        paths = ocrutils.save_ocr_images(request.FILES.iteritems(),
                temp=True, user=request.user.username,
                name=tasktype.lower())
    except AppException, err:
        return HttpResponse(simplejson.dumps({"error": err.message}),
            mimetype="application/json")
    if not paths:
        return HttpResponse(
                simplejson.dumps({"error": "no valid images found"}),
                mimetype="application/json")

    # wrangle the params - this needs improving
    userparams = _get_best_params(request.POST.copy())

    # create a batch db job
    batch_name = request.POST.get("name", "%s %s" % (tasktype, datetime.now()))
    batch = OcrBatch(user=request.user, name=batch_name,
            task_type=celerytask.name, batch_type="MULTI")
    batch.save()

    subtasks = []
    try:
        for path in paths:
            tid = ocrutils.get_new_task_id(path)
            ocrtask = OcrTask(task_id=tid, batch=batch,
                    page_name=os.path.basename(path), status="INIT")
            ocrtask.save()
            subtasks.append(
                celerytask.subtask(
                    args=(path.encode(), userparams),
                    options=dict(task_id=tid, loglevel=60, retries=2)))
        tasksetresults = TaskSet(tasks=subtasks).apply_async()
        out = {
            "job_name": tasksetresults.taskset_id,
            "count" : tasksetresults.total,
            "subtasks" : [t.task_id for t in tasksetresults.subtasks],
            "completed_count": tasksetresults.completed_count(),
            "done": tasksetresults.successful() | tasksetresults.failed(),
        }
    except Exception, e:
        transaction.rollback()
        print e.message
        return HttpResponse(e.message,
                mimetype="application/json")

    transaction.commit()
    return HttpResponse(simplejson.dumps(out),
            mimetype="application/json")

@login_required
def batch_results(request, job_name):
    """
    Get results for a taskset.
    """
    subtasks = request.POST.getlist("task_id")
    asynctasks = [celeryresult.AsyncResult(t) for t in subtasks]
    tasksetresults = celeryresult.TaskSetResult(job_name, asynctasks)
    out = {
        "job_name": tasksetresults.taskset_id,
        "count" : tasksetresults.total,
        "completed_count": tasksetresults.completed_count(),
        "done": tasksetresults.successful() | tasksetresults.failed(),
    }
    return HttpResponse(simplejson.dumps(out),
            mimetype="application/json")

@login_required
@transaction.commit_manually
def binarize(request):
    """
        Save a posted image to the DFS.  Binarize it with Celery.
    """

    return _ocr_task(
        request,
        "ocr/binarize.html",
        {},
        "Binarize",
        tasks.BinarizePageTask,
    )

@login_required
@transaction.commit_manually
def convert(request):
    """
    Save a posted image to the DFS and convert it with Celery.
    """

    # add available seg and bin presets to the context
    context = {
        "binpresets": OcrPreset.objects.filter(
            type="binarize").order_by("name"),
        "segpresets": OcrPreset.objects.filter(
            type="segment").order_by("name"),
    }

    return _ocr_task(
        request,
        "ocr/convert.html",
        context,
        "Convert",
        tasks.ConvertPageTask,
    )

@login_required
@transaction.commit_manually
def segment(request):
    """
        Save a posted image to the DFS.  Segment it with Celery.
    """

    # add available seg and bin presets to the context
    context = {
        "binpresets": OcrPreset.objects.filter(
            type="binarize").order_by("name"),
    }

    return _ocr_task(
        request,
        "ocr/segment.html",
        context,
        "Segment",
        tasks.SegmentPageTask,
    )

@login_required
def results(request, job_name):
    """
    Retrieve the results using the previously provided task name.
    """
    async = celeryresult.AsyncResult(job_name)
    if async is None:
        raise Http404

    return _format_task_results(request, async)

@login_required
def components(request):
    """
    List OCRopus components - either all or those
    of an optional type.
    """
    comps = ocrutils.get_ocropus_components(
            request.GET.getlist("type"), request.GET.getlist("name"))
    return HttpResponse(simplejson.dumps(comps), mimetype="application/json")

def test(request):
    """
    Dummy action for running JS unit tests.  Probably needs to
    be put somewhere else.
    """
    return render_to_response("ocr/test.html", {})

def _ocr_task(request, template, context, tasktype, celerytask):
    """
    Generic handler for OCR tasks which run a celery job.
    """
    if not request.method == "POST":
        return render_to_response(template, context,
                        context_instance=RequestContext(request))

    # save our files to the DFS and return a list of addresses
    if request.POST.get("png"):
        paths = [ocrutils.media_url_to_path(request.POST.get("png"))]
    else:
        try:
            paths = ocrutils.save_ocr_images(request.FILES.iteritems(),
                    temp=True, user=request.user.username,
                    name=tasktype.lower())
        except AppException, err:
            return HttpResponse(simplejson.dumps({"error": err.message}),
                mimetype="application/json")
    if not paths:
        return HttpResponse(
                simplejson.dumps({"error": "no valid images found"}),
                mimetype="application/json")

    # wrangle the params - this needs improving
    userparams = _get_best_params(request.POST.copy())

    # create a batch db job
    batch = OcrBatch(user=request.user, name="%s %s" % (tasktype, datetime.now()),
            task_type=celerytask.name, batch_type="ONESHOT")
    batch.save()

    # init the job from our params
    asynctasks = []
    for path in paths:
        tid = ocrutils.get_new_task_id(path)
        ocrtask = OcrTask(task_id=tid, batch=batch,
                page_name=os.path.basename(path), status="INIT")
        ocrtask.save()
        asynctasks.append(
            celerytask.apply_async(
                args=(path.encode(), userparams),
                    task_id=tid, loglevel=60, retries=2))
    try:
        # aggregate the results
        out = []
        for async in asynctasks:
            result = async.wait() if _should_wait(request) else async.result
            out.append({
                "job_name": async.task_id,
                "status": async.status,
                "results": result,
            })
        # should be past the danger zone now...
        transaction.commit()
        return _json_or_text_response(request, out)

    except Exception, err:
        print err
        transaction.rollback()
        return HttpResponse(
            simplejson.dumps({
                "error": err.message,
                "trace": "\n".join(
                    [ "\t".join(str(t)) for t in traceback.extract_stack()]
                )
            }),
            mimetype="application/json"
        )

def _format_task_results(request, async):
    """
    Wrap the results in JSON metadata.  Treat
    exceptions as a special case.
    """
    if async.status == "FAILURE":
        err = async.result
        taskmeta = OcrTask.objects.get(task_id=async.task_id)
        return HttpResponse(
            simplejson.dumps({
                "error": err.message,
                "trace": taskmeta.traceback
            }),
            mimetype="application/json"
        )
    return _json_or_text_response(
        request, {
            "job_name": async.task_id,
            "status": async.status,
            "results": async.result
        }
    )

def _wants_text_format(request):
    """
    Determine whether we should send back plain text instead of JSON.
    """
    return request.META.get("HTTP_ACCEPT", "") == "text/plain" \
        or request.GET.get("format", "") == "text" \
        or request.POST.get("format", "") == "text"

def _wants_png_format(request):
    """
    Determine whether to send back an image rather than JSON.
    (For binarisation/segmentation.
    """
    return (request.META.get("HTTP_ACCEPT", "") == "image/png" \
        or request.GET.get("format", "") == "png" \
        or request.POST.get("format", "") == "png")

def _should_wait(request):
    """
    Determine if we should block waiting for the results.  The
    default is to not block.
    """
    return request.GET.get("wait", False)

def _json_or_text_response(request, json):
    """
    Format the output string accordingly.
    """
    mimetype = "application/json"

    # bad code alert!  if uploaded via the dodgy
    # Ajax iframe method, force a text response
    # otherwise firefox eats the results for us...
    if request.POST.get("_iframe"):
        mimetype = "text/html"

    if _wants_png_format(request):
        if isinstance(json, dict):
            json = [json]
        mimetype = "image/png"
        try:
            path = ocrutils.media_url_to_path(
                    json[0]["results"]["out"])
            result = open(path, "rb").read()
        except (IndexError, KeyError), err:
            result = ""
            raise err
    elif _wants_text_format(request):
        if isinstance(json, dict):
            json = [json]
        mimetype = "text/plain"
        result = ""
        for page in json:
            if page.get("results"):
                result += ocrutils.output_to_plain_text(page.get("results"))
                result += "\n"
    else:
        result = simplejson.dumps(json)
    return HttpResponse(result, mimetype=mimetype)

def _get_preset_data(param):
    """
    Fetch a preset by primary key and return its data
    dict for merging into OCR params.  Try to get the
    preset via primary key (as used via the web UI)
    or name (as used via Curl).
    """
    pmatch = re.match("preset_(\d+)", param)
    data = None
    #keyval = {}
    if pmatch:
        keyval = dict(pk=pmatch.groups()[0])
    else:
        keyval = dict(name=param)

    try:
        preset = OcrPreset.objects.get(**keyval)
        data = preset.data
    except OcrPreset.DoesNotExist:
        pass
    return data

def _get_best_params(postdict):
    """
    Attempt to determine the best params if not specified in
    POST.  This is contingent on data in the models table.
    TODO: Make this less horrible
    """

    userparams = postdict
    userparams["engine"] = postdict.get("engine", "tesseract")

    # get the bin and seg params.  These are either dicts or default strings
    segparam = postdict.get("psegmenter", "SegmentPageByRAST")
    segdata = _get_preset_data(segparam)
    if segdata:
        userparams.update(segdata)
    else:
        userparams["psegmenter"] = segparam

    binparam = postdict.get("clean", "StandardPreprocessing")
    bindata = _get_preset_data(binparam)
    if bindata:
        userparams.update(bindata)
    else:
        userparams["clean"] = binparam

    # get the lmodel/cmodel, either model object paths or defaults
    for modparam in ("cmodel", "lmodel"):
        try:
            model = OcrModel.objects.get(name=userparams.get(modparam, "???"))
            userparams[modparam] = model.file.path
        except OcrModel.DoesNotExist:
            # try and choose the best model accordingly - this is a model
            # named "Default Something"
            modtype = "char" if modparam == "cmodel" else "lang"
            try:
                model = OcrModel.objects.filter(
                    name__icontains="default",
                    app__iexact=userparams["engine"],
                    type__iexact=modtype,
                )[0]
                userparams[modparam] = model.file.path
            except IndexError:
                userparams[modparam] = "???"

    return userparams
<commit_msg>Include subtasks when returning batch results.  Still needs serious rethinking<commit_after>"""
Basic OCR functions.  Submit OCR tasks and retrieve the result.
"""

import re
import os
import traceback
from datetime import datetime
from celery import result as celeryresult
from celery.task.sets import TaskSet, subtask
from django.conf import settings
from django.contrib.auth.decorators import login_required
from django.db import transaction
from django.http import HttpResponse, HttpResponseRedirect, Http404
from django.shortcuts import render_to_response
from django.template import RequestContext
from django.utils import simplejson
from ocradmin.ocr import tasks
from ocradmin.ocr import utils as ocrutils
from ocradmin.ocrmodels.models import OcrModel
from ocradmin.ocrpresets.models import OcrPreset
from ocradmin.ocrtasks.models import OcrTask, OcrBatch

class AppException(StandardError):
    """
    Most generic app error.
    """
    pass

@login_required
def index(request):
    """
    OCR index page.
    """
    return HttpResponseRedirect("/ocr/convert/")

@login_required
@transaction.commit_manually
def batch(request):
    """
    Save a batch of posted image to the DFS and convert them with Celery.
    """

    # add available seg and bin presets to the context
    context = {
        "binpresets": OcrPreset.objects.filter(
            type="binarize").order_by("name"),
        "segpresets": OcrPreset.objects.filter(
            type="segment").order_by("name"),
    }
    tasktype = "Batch"
    celerytask = tasks.ConvertPageTask

    if not request.method == "POST":
        return render_to_response("ocr/batch.html", context,
                        context_instance=RequestContext(request))

    try:
        paths = ocrutils.save_ocr_images(request.FILES.iteritems(),
                temp=True, user=request.user.username,
                name=tasktype.lower())
    except AppException, err:
        return HttpResponse(simplejson.dumps({"error": err.message}),
            mimetype="application/json")
    if not paths:
        return HttpResponse(
                simplejson.dumps({"error": "no valid images found"}),
                mimetype="application/json")

    # wrangle the params - this needs improving
    userparams = _get_best_params(request.POST.copy())

    # create a batch db job
    batch_name = request.POST.get("name", "%s %s" % (tasktype, datetime.now()))
    batch = OcrBatch(user=request.user, name=batch_name,
            task_type=celerytask.name, batch_type="MULTI")
    batch.save()

    subtasks = []
    try:
        for path in paths:
            tid = ocrutils.get_new_task_id(path)
            ocrtask = OcrTask(task_id=tid, batch=batch,
                    page_name=os.path.basename(path), status="INIT")
            ocrtask.save()
            subtasks.append(
                celerytask.subtask(
                    args=(path.encode(), userparams),
                    options=dict(task_id=tid, loglevel=60, retries=2)))
        tasksetresults = TaskSet(tasks=subtasks).apply_async()
        out = {
            "job_name": tasksetresults.taskset_id,
            "count" : tasksetresults.total,
            "subtasks" : [t.task_id for t in tasksetresults.subtasks],
            "completed_count": tasksetresults.completed_count(),
            "done": tasksetresults.successful() | tasksetresults.failed(),
        }
    except Exception, e:
        transaction.rollback()
        print e.message
        return HttpResponse(e.message,
                mimetype="application/json")

    transaction.commit()
    return HttpResponse(simplejson.dumps(out),
            mimetype="application/json")

@login_required
def batch_results(request, job_name):
    """
    Get results for a taskset.
    """
    subtasks = request.POST.getlist("task_id")
    asynctasks = [celeryresult.AsyncResult(t) for t in subtasks]
    tasksetresults = celeryresult.TaskSetResult(job_name, asynctasks)
    out = {
        "job_name": tasksetresults.taskset_id,
        "count" : tasksetresults.total,
        "subtasks" : [t.task_id for t in tasksetresults.subtasks],
        "completed_count": tasksetresults.completed_count(),
        "done": tasksetresults.successful() | tasksetresults.failed(),
    }
    return HttpResponse(simplejson.dumps(out),
            mimetype="application/json")

@login_required
@transaction.commit_manually
def binarize(request):
    """
        Save a posted image to the DFS.  Binarize it with Celery.
    """

    return _ocr_task(
        request,
        "ocr/binarize.html",
        {},
        "Binarize",
        tasks.BinarizePageTask,
    )

@login_required
@transaction.commit_manually
def convert(request):
    """
    Save a posted image to the DFS and convert it with Celery.
    """

    # add available seg and bin presets to the context
    context = {
        "binpresets": OcrPreset.objects.filter(
            type="binarize").order_by("name"),
        "segpresets": OcrPreset.objects.filter(
            type="segment").order_by("name"),
    }

    return _ocr_task(
        request,
        "ocr/convert.html",
        context,
        "Convert",
        tasks.ConvertPageTask,
    )

@login_required
@transaction.commit_manually
def segment(request):
    """
        Save a posted image to the DFS.  Segment it with Celery.
    """

    # add available seg and bin presets to the context
    context = {
        "binpresets": OcrPreset.objects.filter(
            type="binarize").order_by("name"),
    }

    return _ocr_task(
        request,
        "ocr/segment.html",
        context,
        "Segment",
        tasks.SegmentPageTask,
    )

@login_required
def results(request, job_name):
    """
    Retrieve the results using the previously provided task name.
    """
    async = celeryresult.AsyncResult(job_name)
    if async is None:
        raise Http404

    return _format_task_results(request, async)

@login_required
def components(request):
    """
    List OCRopus components - either all or those
    of an optional type.
    """
    comps = ocrutils.get_ocropus_components(
            request.GET.getlist("type"), request.GET.getlist("name"))
    return HttpResponse(simplejson.dumps(comps), mimetype="application/json")

def test(request):
    """
    Dummy action for running JS unit tests.  Probably needs to
    be put somewhere else.
    """
    return render_to_response("ocr/test.html", {})

def _ocr_task(request, template, context, tasktype, celerytask):
    """
    Generic handler for OCR tasks which run a celery job.
    """
    if not request.method == "POST":
        return render_to_response(template, context,
                        context_instance=RequestContext(request))

    # save our files to the DFS and return a list of addresses
    if request.POST.get("png"):
        paths = [ocrutils.media_url_to_path(request.POST.get("png"))]
    else:
        try:
            paths = ocrutils.save_ocr_images(request.FILES.iteritems(),
                    temp=True, user=request.user.username,
                    name=tasktype.lower())
        except AppException, err:
            return HttpResponse(simplejson.dumps({"error": err.message}),
                mimetype="application/json")
    if not paths:
        return HttpResponse(
                simplejson.dumps({"error": "no valid images found"}),
                mimetype="application/json")

    # wrangle the params - this needs improving
    userparams = _get_best_params(request.POST.copy())

    # create a batch db job
    batch = OcrBatch(user=request.user, name="%s %s" % (tasktype, datetime.now()),
            task_type=celerytask.name, batch_type="ONESHOT")
    batch.save()

    # init the job from our params
    asynctasks = []
    for path in paths:
        tid = ocrutils.get_new_task_id(path)
        ocrtask = OcrTask(task_id=tid, batch=batch,
                page_name=os.path.basename(path), status="INIT")
        ocrtask.save()
        asynctasks.append(
            celerytask.apply_async(
                args=(path.encode(), userparams),
                    task_id=tid, loglevel=60, retries=2))
    try:
        # aggregate the results
        out = []
        for async in asynctasks:
            result = async.wait() if _should_wait(request) else async.result
            out.append({
                "job_name": async.task_id,
                "status": async.status,
                "results": result,
            })
        # should be past the danger zone now...
        transaction.commit()
        return _json_or_text_response(request, out)

    except Exception, err:
        print err
        transaction.rollback()
        return HttpResponse(
            simplejson.dumps({
                "error": err.message,
                "trace": "\n".join(
                    [ "\t".join(str(t)) for t in traceback.extract_stack()]
                )
            }),
            mimetype="application/json"
        )

def _format_task_results(request, async):
    """
    Wrap the results in JSON metadata.  Treat
    exceptions as a special case.
    """
    if async.status == "FAILURE":
        err = async.result
        taskmeta = OcrTask.objects.get(task_id=async.task_id)
        return HttpResponse(
            simplejson.dumps({
                "error": err.message,
                "trace": taskmeta.traceback
            }),
            mimetype="application/json"
        )
    return _json_or_text_response(
        request, {
            "job_name": async.task_id,
            "status": async.status,
            "results": async.result
        }
    )

def _wants_text_format(request):
    """
    Determine whether we should send back plain text instead of JSON.
    """
    return request.META.get("HTTP_ACCEPT", "") == "text/plain" \
        or request.GET.get("format", "") == "text" \
        or request.POST.get("format", "") == "text"

def _wants_png_format(request):
    """
    Determine whether to send back an image rather than JSON.
    (For binarisation/segmentation.
    """
    return (request.META.get("HTTP_ACCEPT", "") == "image/png" \
        or request.GET.get("format", "") == "png" \
        or request.POST.get("format", "") == "png")

def _should_wait(request):
    """
    Determine if we should block waiting for the results.  The
    default is to not block.
    """
    return request.GET.get("wait", False)

def _json_or_text_response(request, json):
    """
    Format the output string accordingly.
    """
    mimetype = "application/json"

    # bad code alert!  if uploaded via the dodgy
    # Ajax iframe method, force a text response
    # otherwise firefox eats the results for us...
    if request.POST.get("_iframe"):
        mimetype = "text/html"

    if _wants_png_format(request):
        if isinstance(json, dict):
            json = [json]
        mimetype = "image/png"
        try:
            path = ocrutils.media_url_to_path(
                    json[0]["results"]["out"])
            result = open(path, "rb").read()
        except (IndexError, KeyError), err:
            result = ""
            raise err
    elif _wants_text_format(request):
        if isinstance(json, dict):
            json = [json]
        mimetype = "text/plain"
        result = ""
        for page in json:
            if page.get("results"):
                result += ocrutils.output_to_plain_text(page.get("results"))
                result += "\n"
    else:
        result = simplejson.dumps(json)
    return HttpResponse(result, mimetype=mimetype)

def _get_preset_data(param):
    """
    Fetch a preset by primary key and return its data
    dict for merging into OCR params.  Try to get the
    preset via primary key (as used via the web UI)
    or name (as used via Curl).
    """
    pmatch = re.match("preset_(\d+)", param)
    data = None
    #keyval = {}
    if pmatch:
        keyval = dict(pk=pmatch.groups()[0])
    else:
        keyval = dict(name=param)

    try:
        preset = OcrPreset.objects.get(**keyval)
        data = preset.data
    except OcrPreset.DoesNotExist:
        pass
    return data

def _get_best_params(postdict):
    """
    Attempt to determine the best params if not specified in
    POST.  This is contingent on data in the models table.
    TODO: Make this less horrible
    """

    userparams = postdict
    userparams["engine"] = postdict.get("engine", "tesseract")

    # get the bin and seg params.  These are either dicts or default strings
    segparam = postdict.get("psegmenter", "SegmentPageByRAST")
    segdata = _get_preset_data(segparam)
    if segdata:
        userparams.update(segdata)
    else:
        userparams["psegmenter"] = segparam

    binparam = postdict.get("clean", "StandardPreprocessing")
    bindata = _get_preset_data(binparam)
    if bindata:
        userparams.update(bindata)
    else:
        userparams["clean"] = binparam

    # get the lmodel/cmodel, either model object paths or defaults
    for modparam in ("cmodel", "lmodel"):
        try:
            model = OcrModel.objects.get(name=userparams.get(modparam, "???"))
            userparams[modparam] = model.file.path
        except OcrModel.DoesNotExist:
            # try and choose the best model accordingly - this is a model
            # named "Default Something"
            modtype = "char" if modparam == "cmodel" else "lang"
            try:
                model = OcrModel.objects.filter(
                    name__icontains="default",
                    app__iexact=userparams["engine"],
                    type__iexact=modtype,
                )[0]
                userparams[modparam] = model.file.path
            except IndexError:
                userparams[modparam] = "???"

    return userparams
<|endoftext|>