<commit_before>#------------------------------------------------------------------------------
# Copyright (c) 2007, Enthought, Inc.
# All rights reserved.
#
# This software is provided without warranty under the terms of the BSD license
# available at http://www.enthought.com/licenses/BSD.txt and may be
# redistributed only under the conditions described in the aforementioned
# license.
#
# Rick Ratzel - 2006-06-27
#------------------------------------------------------------------------------

import sys
import optparse
import site
import os
from os import path

from enthought.enstaller.downloader import \
     Downloader
from enthought.enstaller.api import \
     ENTHOUGHT_REPO, get_app_version_string
from enthought.enstaller.session import \
     Session
from enthought.enstaller.cli import \
     CLI
from enthought.enstaller.logger import \
     Logger

#
# Set a flag indicating if the Enstaller GUI was installed.
#
try :
    import enthought.enstaller.gui
    HAVE_GUI = True
except ImportError :
    HAVE_GUI = False

#
# Normally, ETS.application_home is set properly, but since Enstaller may be
# started by a script that is not in the application dir (application_home is
# based on the dirname of the dir containing the startup script), manually set
# the application_home here for other modules to use.
#
from enthought.ets.api import \
     ETS
ETS.application_home = path.join( ETS.application_data, "enstaller" )

def build_option_parser( program_name=sys.argv[0] ) :
    """
    Returns an OptionParser instance for use with the Enstaller standalone app.
    """
    usage = "USAGE: %prog [options]"

    #
    # Add a new link only if it has not been added before.
    #
    def add_link( option, opt_str, value, parser ) :
        if( len( parser.rargs ) > 0 ) :
            arg = parser.rargs[0]
            if( not( arg.startswith( "-" ) ) ) :
                if( not( arg in parser.values.find_links ) ) :
                    parser.values.find_links.append( arg )
                del parser.rargs[0]

    #
    # Define a callback function for the opt_parser which makes sure only one
    # action at a time is performed and that a package name was given if needed.
    #
    all_actions = ["list_installed", "list_repos", "remove", "upgrade",
                   "list_upgrades", "activate", "deactivate"]
    need_package_actions = ["--remove", "--activate", "--deactivate"]

    def check_valid_action( option, opt_str, value, parser ) :
        setattr( parser.values, option.dest, True )
        actions = [getattr( parser.values, a ) for a in all_actions]
        #
        # Check that only one action was given
        #
        if( actions.count( True ) > 1 ) :
            msg = "Cannot combine Enstaller actions. Please specify only one of:"
            msg += " %s" % ", ".join( ['"--%s"' % a for a in all_actions] )
            raise optparse.OptionValueError( msg )
        #
        # Check if at least one package was given, if needed
        #
        if( option.get_opt_string() in need_package_actions ) :
            found_package_spec = False
            for arg in (parser.largs + parser.rargs) :
                if( not( arg.startswith( "-" ) ) ) :
                    found_package_spec = True
                    break
            if( not( found_package_spec ) ) :
                msg = "Must specify at least one package with the: " + \
                      "%s action" % opt_str
                raise optparse.OptionValueError( msg )

    #
    # Create an OptionParser and initialize with Enstaller options.
    #
    opt_parser = optparse.OptionParser( prog=program_name, usage=usage,
                                        version=get_app_version_string() )

    opt_parser.add_option( "-c", "--command-line",
                           dest="gui", default=True,
                           action="store_false",
                           help="do not use the Enstaller GUI" )

    opt_parser.add_option( "-d", "--install-dir",
                           dest="install_dir", metavar="<dir>",
                           default="",
                           help="use an alternate directory to install" + \
                           "packages to (defaults to site-packages for" + \
                           "use by all users)" )

    opt_parser.add_option( "-f", "--find-links",
                           dest="find_links", metavar="<repo>",
                           default=[],
                           action="callback", callback=add_link,
                           help="add a package repository URL to the " + \
                           "search list" )

    opt_parser.add_option( "-t", "--batch",
                           dest="prompting", default=True,
                           action="store_false",
                           help="batch mode - do not confirm operations " + \
                           "(command-line only)" )

    opt_parser.add_option( "-v", "--verbose",
                           dest="verbose",default=False,
                           action="store_true",
                           help="print debug-level messages" )

    opt_parser.add_option( "--no-default-enthought-repo",
                           dest="use_default_enthought_repo", default=True,
                           action="store_false",
                           help="do not use the Enthought repository " + \
                           "by default" )

    opt_parser.add_option( "-l", "--list-installed",
                           dest="list_installed", default=False,
                           action="callback", callback=check_valid_action,
                           help="print information about a package or " + \
                           "packages installed (default is all packages)" )

    opt_parser.add_option( "-L", "--list-repos",
                           dest="list_repos", default=False,
                           action="callback", callback=check_valid_action,
                           help="print information about a package or " + \
                           "packages available from the repositries " + \
                           "(default is all available packages)" )

    opt_parser.add_option( "-r", "--remove",
                           dest="remove", default=False,
                           action="callback", callback=check_valid_action,
                           help="remove a package or packages installed" )

    opt_parser.add_option( "-U", "--upgrade",
                           dest="upgrade", default=False,
                           action="callback", callback=check_valid_action,
                           help="upgrade a package or packages installed " + \
                           "(default is all packages)" )

    opt_parser.add_option( "-u", "--list-upgrades",
                           dest="list_upgrades", default=False,
                           action="callback", callback=check_valid_action,
                           help="list the upgrades available to a package " + \
                           "or packages installed (default is all packages)" )

    opt_parser.add_option( "-A", "--activate",
                           dest="activate", default=False,
                           action="callback", callback=check_valid_action,
                           help="activate a package or packages installed" )

    opt_parser.add_option( "-D", "--deactivate",
                           dest="deactivate", default=False,
                           action="callback", callback=check_valid_action,
                           help="deactivate a package or packages installed" )

    #
    # Override the optparse check_values method in order to add the default
    # Enthought repo last in the order of find_links precedence, if it is to
    # be used at all.
    #
    def check_values( values, args ) :
        find_links = values.find_links
        use_def_en_repo = values.use_default_enthought_repo
        if( use_def_en_repo and not( ENTHOUGHT_REPO in find_links ) ) :
            find_links.append( ENTHOUGHT_REPO )
        return (values, args)

    opt_parser.check_values = check_values

    return opt_parser

def override_opt_parse_output( opt_parser, logging_handle ) :
    """
    Replaces the methods on the opt_parser that write to stdout/stderr with ones
    that write to the logging_handle instead.
    This should not be called on opt_parser instances which have custom error()
    or print_usage() methods!
    """

    def error( msg ) :
        opt_parser.print_usage( logging_handle )
        opt_parser.exit( 2, "%s: error: %s\n" \
                         % (opt_parser.get_prog_name(), msg) )

    def print_usage( file=None ) :
        if( opt_parser.usage ) :
            print >>logging_handle, opt_parser.get_usage()

    def print_help( file=None ) :
        logging_handle.write( opt_parser.format_help() )

    def exit( status=0, msg=None ) :
        if( msg ) :
            logging_handle.write( msg )
        sys.exit( status )

    def print_version( file=None ) :
        if( opt_parser.version ) :
            print >>logging_handle, opt_parser.get_version()

    opt_parser.error = error
    opt_parser.print_usage = print_usage
    opt_parser.print_help = print_help
    opt_parser.exit = exit
    opt_parser.print_version = print_version

def postprocess_args( opt_parser, options, package_specs, logging_handle ) :
    """
    Checks the complete set of args and checks that they make sense for this
    particular application.  Also removes the program name from the list of
    package_specs
    """

    retcode = 0

    #
    # Sometimes, missing args to -d will cause it to treat the next option
    # as the install directory...bug in optparse?
    #
    if( options.install_dir.startswith( "-" ) ) :
        logging_handle.write( "Error: Bad install directory: %s\n" \
                              % options.install_dir )
        retcode = 2

    cli_options_given = (len( package_specs ) > 0) or options.list_installed or \
                        options.list_repos or options.remove or \
                        options.upgrade or options.list_upgrades or \
                        options.activate or options.deactivate

    cli_mode = not( HAVE_GUI ) or cli_options_given

    #
    # override the GUI default if cli args were given.
    #
    if( cli_mode ) :
        options.gui = False
    #
    # The --install-dir option can only be given with the install action
    #
    if( (options.install_dir != "") and
        (options.list_installed or options.list_repos or options.remove or \
         options.activate or options.deactivate) ) :
        logging_handle.write( "Error: The --install-dir (-d) option can " + \
                              "only be used when installing or upgrading " + \
                              "packages.\n" )
        retcode = 2
    #
    # Make sure there is something to do if a GUI is not to be started.
    #
    elif( not( options.gui ) and not( cli_mode ) ) :
        logging_handle.write( "Error: Must provide one or more packages " + \
                              "to install or an action to perform when " + \
                              "not using the Enstaller GUI.\n" )
        retcode = 2
    #
    # Issue a warning if no prompting selected with GUI.
    #
    elif( options.gui and not( options.prompting ) ) :
        logging_handle.write( "Warning: --batch (-t) is ignored when using " + \
                              "the Enstaller GUI.\n" )

    #
    # If the GUI is not installed and no options were given, print help
    #
    elif( not( HAVE_GUI ) and not( cli_options_given ) ) :
        opt_parser.print_help()
        sys.exit( 0 )

    return retcode

def main( argv=sys.argv, logging_handle=sys.stdout ) :
    """
    Starts a new Session, initializing it based on the command-line args passed
    in and logging all output to logging_handle.  This function should not raise
    any exceptions...instead it will return non-0 on error and log a message to
    the logging handle.
    """

    assert (len( argv ) > 0), "argv arg to main cannot be an empty list"
    retcode = 1
    #
    # Build a command-line processor that uses the logging_handle for output.
    #
    opt_parser = build_option_parser( argv[0] )

    #
    # Catch sys.exit to prevent OptionParser from shutting down the app if the
    # args were invalid.
    #
    try :
        (options, package_specs) = opt_parser.parse_args( argv[1:] )
        retcode = postprocess_args( opt_parser, options, package_specs,
                                    logging_handle )

    except SystemExit, exit_code :
        retcode = exit_code

    #
    # If the args were valid, instantiate a session and call methods based on
    # args passed in
    #
    if( retcode == 0 ) :
        #
        # Import here before a session is instantiated but after the module is
        # completely imported so only the GUI option imports GUI dependencies.
        #
        if( options.gui ) :
            from enthought.enstaller.gui.gui import GUI
        #
        # Instantiate a session and a logger, used for both GUI and CLI
        #
        logger = Logger( targets=[logging_handle] )
        session = Session( logging_handle = logger,
                           verbose        = options.verbose,
                           prompting      = options.prompting,
                           find_links     = options.find_links )
        #
        # Scan sys.path...this is needed for all actions.
        #
        session.initialize()
        install_dir = (options.install_dir or Downloader.get_site_packages_dir())

        #
        # Launch either the GUI or the CLI with the appropriate action.
        #
        if( options.gui ) :
            gui = GUI( logging_handle=logger,
                       verbose=options.verbose,
                       session=session )
            gui.install_dir = install_dir
            retcode = gui.show()

        else :
            logger.copy_to_buffer = False
            cli = CLI( logging_handle=logger,
                       verbose=options.verbose,
                       prompting=options.prompting,
                       session=session )

            if( options.list_installed ) :
                retcode = cli.list_installed( package_specs )

            elif( options.list_repos ) :
                retcode = cli.list_repos( package_specs )

            elif( options.remove ) :
                retcode = cli.remove( package_specs )

            elif( options.upgrade ) :
                retcode = cli.upgrade( install_dir, package_specs )

            elif( options.list_upgrades ) :
                retcode = cli.list_upgrades( install_dir, package_specs )

            elif( options.activate ) :
                retcode = cli.activate( package_specs )

            elif( options.deactivate ) :
                retcode = cli.deactivate( package_specs )

            else :
                retcode = cli.install( install_dir, package_specs )

    return retcode

if( __name__ == "__main__" ) :
    sys.exit( main( sys.argv ) )
<commit_msg>* Fix for detecting and activating the GUI when installed as an app egg.<commit_after>#------------------------------------------------------------------------------
# Copyright (c) 2007, Enthought, Inc.
# All rights reserved.
#
# This software is provided without warranty under the terms of the BSD license
# available at http://www.enthought.com/licenses/BSD.txt and may be
# redistributed only under the conditions described in the aforementioned
# license.
#
# Rick Ratzel - 2006-06-27
#------------------------------------------------------------------------------

import sys
import optparse
import site
import os
from os import path
from pkg_resources import \
     require, DistributionNotFound

from enthought.enstaller.downloader import \
     Downloader
from enthought.enstaller.api import \
     ENTHOUGHT_REPO, get_app_version_string, is_standalone_app
from enthought.enstaller.session import \
     Session
from enthought.enstaller.cli import \
     CLI
from enthought.enstaller.logger import \
     Logger

#
# Set a flag indicating if the Enstaller GUI was installed, and handle the
# special case of this module being part of a standalone app Enstaller egg vs.
# a library package egg.
#
HAVE_GUI = True

if( is_standalone_app ) :
    try :
        require( "enstaller.gui" )

    except DistribuitionNotFound :
        HAVE_GUI = False

if( HAVE_GUI ) :
    try :
        import enthought.enstaller.gui

    except ImportError :
        HAVE_GUI = False

#
# Normally, ETS.application_home is set properly, but since Enstaller may be
# started by a script that is not in the application dir (application_home is
# based on the dirname of the dir containing the startup script), manually set
# the application_home here for other modules to use.
#
from enthought.ets.api import \
     ETS
ETS.application_home = path.join( ETS.application_data, "enstaller" )

def build_option_parser( program_name=sys.argv[0] ) :
    """
    Returns an OptionParser instance for use with the Enstaller standalone app.
    """
    usage = "USAGE: %prog [options]"

    #
    # Add a new link only if it has not been added before.
    #
    def add_link( option, opt_str, value, parser ) :
        if( len( parser.rargs ) > 0 ) :
            arg = parser.rargs[0]
            if( not( arg.startswith( "-" ) ) ) :
                if( not( arg in parser.values.find_links ) ) :
                    parser.values.find_links.append( arg )
                del parser.rargs[0]

    #
    # Define a callback function for the opt_parser which makes sure only one
    # action at a time is performed and that a package name was given if needed.
    #
    all_actions = ["list_installed", "list_repos", "remove", "upgrade",
                   "list_upgrades", "activate", "deactivate"]
    need_package_actions = ["--remove", "--activate", "--deactivate"]

    def check_valid_action( option, opt_str, value, parser ) :
        setattr( parser.values, option.dest, True )
        actions = [getattr( parser.values, a ) for a in all_actions]
        #
        # Check that only one action was given
        #
        if( actions.count( True ) > 1 ) :
            msg = "Cannot combine Enstaller actions. Please specify only one of:"
            msg += " %s" % ", ".join( ['"--%s"' % a for a in all_actions] )
            raise optparse.OptionValueError( msg )
        #
        # Check if at least one package was given, if needed
        #
        if( option.get_opt_string() in need_package_actions ) :
            found_package_spec = False
            for arg in (parser.largs + parser.rargs) :
                if( not( arg.startswith( "-" ) ) ) :
                    found_package_spec = True
                    break
            if( not( found_package_spec ) ) :
                msg = "Must specify at least one package with the: " + \
                      "%s action" % opt_str
                raise optparse.OptionValueError( msg )

    #
    # Create an OptionParser and initialize with Enstaller options.
    #
    opt_parser = optparse.OptionParser( prog=program_name, usage=usage,
                                        version=get_app_version_string() )

    opt_parser.add_option( "-c", "--command-line",
                           dest="gui", default=True,
                           action="store_false",
                           help="do not use the Enstaller GUI" )

    opt_parser.add_option( "-d", "--install-dir",
                           dest="install_dir", metavar="<dir>",
                           default="",
                           help="use an alternate directory to install" + \
                           "packages to (defaults to site-packages for" + \
                           "use by all users)" )

    opt_parser.add_option( "-f", "--find-links",
                           dest="find_links", metavar="<repo>",
                           default=[],
                           action="callback", callback=add_link,
                           help="add a package repository URL to the " + \
                           "search list" )

    opt_parser.add_option( "-t", "--batch",
                           dest="prompting", default=True,
                           action="store_false",
                           help="batch mode - do not confirm operations " + \
                           "(command-line only)" )

    opt_parser.add_option( "-v", "--verbose",
                           dest="verbose",default=False,
                           action="store_true",
                           help="print debug-level messages" )

    opt_parser.add_option( "--no-default-enthought-repo",
                           dest="use_default_enthought_repo", default=True,
                           action="store_false",
                           help="do not use the Enthought repository " + \
                           "by default" )

    opt_parser.add_option( "-l", "--list-installed",
                           dest="list_installed", default=False,
                           action="callback", callback=check_valid_action,
                           help="print information about a package or " + \
                           "packages installed (default is all packages)" )

    opt_parser.add_option( "-L", "--list-repos",
                           dest="list_repos", default=False,
                           action="callback", callback=check_valid_action,
                           help="print information about a package or " + \
                           "packages available from the repositries " + \
                           "(default is all available packages)" )

    opt_parser.add_option( "-r", "--remove",
                           dest="remove", default=False,
                           action="callback", callback=check_valid_action,
                           help="remove a package or packages installed" )

    opt_parser.add_option( "-U", "--upgrade",
                           dest="upgrade", default=False,
                           action="callback", callback=check_valid_action,
                           help="upgrade a package or packages installed " + \
                           "(default is all packages)" )

    opt_parser.add_option( "-u", "--list-upgrades",
                           dest="list_upgrades", default=False,
                           action="callback", callback=check_valid_action,
                           help="list the upgrades available to a package " + \
                           "or packages installed (default is all packages)" )

    opt_parser.add_option( "-A", "--activate",
                           dest="activate", default=False,
                           action="callback", callback=check_valid_action,
                           help="activate a package or packages installed" )

    opt_parser.add_option( "-D", "--deactivate",
                           dest="deactivate", default=False,
                           action="callback", callback=check_valid_action,
                           help="deactivate a package or packages installed" )

    #
    # Override the optparse check_values method in order to add the default
    # Enthought repo last in the order of find_links precedence, if it is to
    # be used at all.
    #
    def check_values( values, args ) :
        find_links = values.find_links
        use_def_en_repo = values.use_default_enthought_repo
        if( use_def_en_repo and not( ENTHOUGHT_REPO in find_links ) ) :
            find_links.append( ENTHOUGHT_REPO )
        return (values, args)

    opt_parser.check_values = check_values

    return opt_parser

def override_opt_parse_output( opt_parser, logging_handle ) :
    """
    Replaces the methods on the opt_parser that write to stdout/stderr with ones
    that write to the logging_handle instead.
    This should not be called on opt_parser instances which have custom error()
    or print_usage() methods!
    """

    def error( msg ) :
        opt_parser.print_usage( logging_handle )
        opt_parser.exit( 2, "%s: error: %s\n" \
                         % (opt_parser.get_prog_name(), msg) )

    def print_usage( file=None ) :
        if( opt_parser.usage ) :
            print >>logging_handle, opt_parser.get_usage()

    def print_help( file=None ) :
        logging_handle.write( opt_parser.format_help() )

    def exit( status=0, msg=None ) :
        if( msg ) :
            logging_handle.write( msg )
        sys.exit( status )

    def print_version( file=None ) :
        if( opt_parser.version ) :
            print >>logging_handle, opt_parser.get_version()

    opt_parser.error = error
    opt_parser.print_usage = print_usage
    opt_parser.print_help = print_help
    opt_parser.exit = exit
    opt_parser.print_version = print_version

def postprocess_args( opt_parser, options, package_specs, logging_handle ) :
    """
    Checks the complete set of args and checks that they make sense for this
    particular application.  Also removes the program name from the list of
    package_specs
    """

    retcode = 0

    #
    # Sometimes, missing args to -d will cause it to treat the next option
    # as the install directory...bug in optparse?
    #
    if( options.install_dir.startswith( "-" ) ) :
        logging_handle.write( "Error: Bad install directory: %s\n" \
                              % options.install_dir )
        retcode = 2

    cli_options_given = (len( package_specs ) > 0) or options.list_installed or \
                        options.list_repos or options.remove or \
                        options.upgrade or options.list_upgrades or \
                        options.activate or options.deactivate

    cli_mode = not( HAVE_GUI ) or cli_options_given

    #
    # override the GUI default if cli args were given.
    #
    if( cli_mode ) :
        options.gui = False
    #
    # The --install-dir option can only be given with the install action
    #
    if( (options.install_dir != "") and
        (options.list_installed or options.list_repos or options.remove or \
         options.activate or options.deactivate) ) :
        logging_handle.write( "Error: The --install-dir (-d) option can " + \
                              "only be used when installing or upgrading " + \
                              "packages.\n" )
        retcode = 2
    #
    # Make sure there is something to do if a GUI is not to be started.
    #
    elif( not( options.gui ) and not( cli_mode ) ) :
        logging_handle.write( "Error: Must provide one or more packages " + \
                              "to install or an action to perform when " + \
                              "not using the Enstaller GUI.\n" )
        retcode = 2
    #
    # Issue a warning if no prompting selected with GUI.
    #
    elif( options.gui and not( options.prompting ) ) :
        logging_handle.write( "Warning: --batch (-t) is ignored when using " + \
                              "the Enstaller GUI.\n" )

    #
    # If the GUI is not installed and no options were given, print help
    #
    elif( not( HAVE_GUI ) and not( cli_options_given ) ) :
        opt_parser.print_help()
        sys.exit( 0 )

    return retcode

def main( argv=sys.argv, logging_handle=sys.stdout ) :
    """
    Starts a new Session, initializing it based on the command-line args passed
    in and logging all output to logging_handle.  This function should not raise
    any exceptions...instead it will return non-0 on error and log a message to
    the logging handle.
    """

    assert (len( argv ) > 0), "argv arg to main cannot be an empty list"
    retcode = 1
    #
    # Build a command-line processor that uses the logging_handle for output.
    #
    opt_parser = build_option_parser( argv[0] )

    #
    # Catch sys.exit to prevent OptionParser from shutting down the app if the
    # args were invalid.
    #
    try :
        (options, package_specs) = opt_parser.parse_args( argv[1:] )
        retcode = postprocess_args( opt_parser, options, package_specs,
                                    logging_handle )

    except SystemExit, exit_code :
        retcode = exit_code

    #
    # If the args were valid, instantiate a session and call methods based on
    # args passed in
    #
    if( retcode == 0 ) :
        #
        # Import here before a session is instantiated but after the module is
        # completely imported so only the GUI option imports GUI dependencies.
        #
        if( options.gui ) :
            from enthought.enstaller.gui.gui import GUI
        #
        # Instantiate a session and a logger, used for both GUI and CLI
        #
        logger = Logger( targets=[logging_handle] )
        session = Session( logging_handle = logger,
                           verbose        = options.verbose,
                           prompting      = options.prompting,
                           find_links     = options.find_links )
        #
        # Scan sys.path...this is needed for all actions.
        #
        session.initialize()
        install_dir = (options.install_dir or Downloader.get_site_packages_dir())

        #
        # Launch either the GUI or the CLI with the appropriate action.
        #
        if( options.gui ) :
            gui = GUI( logging_handle=logger,
                       verbose=options.verbose,
                       session=session )
            gui.install_dir = install_dir
            retcode = gui.show()

        else :
            logger.copy_to_buffer = False
            cli = CLI( logging_handle=logger,
                       verbose=options.verbose,
                       prompting=options.prompting,
                       session=session )

            if( options.list_installed ) :
                retcode = cli.list_installed( package_specs )

            elif( options.list_repos ) :
                retcode = cli.list_repos( package_specs )

            elif( options.remove ) :
                retcode = cli.remove( package_specs )

            elif( options.upgrade ) :
                retcode = cli.upgrade( install_dir, package_specs )

            elif( options.list_upgrades ) :
                retcode = cli.list_upgrades( install_dir, package_specs )

            elif( options.activate ) :
                retcode = cli.activate( package_specs )

            elif( options.deactivate ) :
                retcode = cli.deactivate( package_specs )

            else :
                retcode = cli.install( install_dir, package_specs )

    return retcode

if( __name__ == "__main__" ) :
    sys.exit( main( sys.argv ) )
<|endoftext|>


<commit_before>import os
import time
import sqlite3
import nanotime

from dvc.config import Config
from dvc.system import System
from dvc.utils import file_md5, remove
from dvc.exceptions import DvcException
from dvc.logger import Logger

class StateDuplicateError(DvcException):
    pass

class State(object):
    VERSION = 1
    STATE_FILE = 'state'
    STATE_TABLE = 'state'
    STATE_TABLE_LAYOUT = "inode UNSIGNED INTEGER PRIMARY KEY, " \
                         "mtime TEXT NOT NULL, " \
                         "md5 TEXT NOT NULL, " \
                         "timestamp TEXT NOT NULL"

    STATE_INFO_TABLE = 'state_info'
    STATE_INFO_TABLE_LAYOUT = 'count INTEGER'
    STATE_INFO_ROW = 1

    LINK_STATE_TABLE = 'link_state'
    LINK_STATE_TABLE_LAYOUT = "path TEXT PRIMARY KEY, " \
                              "inode UNSIGNED INTEGER NOT NULL, " \
                              "mtime TEXT NOT NULL"

    STATE_ROW_LIMIT = 10000000
    STATE_ROW_CLEANUP_QUOTA = 50

    def __init__(self, project, config):
        self.project = project
        self.dvc_dir = project.dvc_dir
        self.root_dir = project.root_dir

        self.row_limit = 100
        self.row_cleanup_quota = 50

        c = config.get(Config.SECTION_STATE, {})
        self.row_limit = c.get(Config.SECTION_STATE_ROW_LIMIT,
                               self.STATE_ROW_LIMIT)
        self.row_cleanup_quota = c.get(Config.SECTION_STATE_ROW_CLEANUP_QUOTA,
                                       self.STATE_ROW_CLEANUP_QUOTA)

        if not self.dvc_dir:
            self.state_file = None
            return

        self.state_file = os.path.join(self.dvc_dir, self.STATE_FILE)
        self.db = None
        self.c = None
        self.inserts = 0

    def __enter__(self):
        self.load()

    def __exit__(self, type, value, tb):
        self.dump()

    @staticmethod
    def init(project):
        return State(project)

    def _collect(self, path):
        if os.path.isdir(path):
            return self.project.cache.local.collect_dir_cache(path)
        else:
            return (file_md5(path)[0], None)

    def changed(self, path, md5):
        actual = self.update(path)

        if not md5 or not actual:
            return True

        return actual.split('.')[0] != md5.split('.')[0]

    def _execute(self, cmd):
        Logger.debug(cmd)
        return self.c.execute(cmd)

    def _fetchall(self):
        ret = self.c.fetchall()
        Logger.debug("fetched: {}".format(ret))
        return ret

    def _load(self, empty=False):
        from dvc import VERSION

        if not empty:
            cmd = "PRAGMA user_version;"
            self._execute(cmd)
            ret = self._fetchall()
            assert len(ret) == 1
            assert len(ret[0]) == 1
            assert isinstance(ret[0][0], int)
            version = ret[0][0]

            if version > self.VERSION:
                msg = "You are using an old version '{}' of dvc that is " \
                      "using state file version '{}' which is not " \
                      "compatible with the state file version '{}' that " \
                      "is used in this projet. Please upgrade right now!"
                raise DvcException(msg.format(VERSION,
                                              self.VERSION,
                                              version))
            elif version < self.VERSION:
                msg = "State file version '{}' is too old. " \
                      "Reformatting to the current version '{}'."
                self.project.logger.warn(msg.format(version, self.VERSION))
                cmd = "DROP TABLE IF EXISTS {};"
                self._execute(cmd.format(self.STATE_TABLE))
                self._execute(cmd.format(self.STATE_INFO_TABLE))
                self._execute(cmd.format(self.LINK_STATE_TABLE))

        # Check that the state file is indeed a database
        cmd = "CREATE TABLE IF NOT EXISTS {} ({})"
        self._execute(cmd.format(self.STATE_TABLE,
                                 self.STATE_TABLE_LAYOUT))
        self._execute(cmd.format(self.STATE_INFO_TABLE,
                                 self.STATE_INFO_TABLE_LAYOUT))
        self._execute(cmd.format(self.LINK_STATE_TABLE,
                                 self.LINK_STATE_TABLE_LAYOUT))

        cmd = "INSERT OR IGNORE INTO {} (count) SELECT 0 " \
              "WHERE NOT EXISTS (SELECT * FROM {})"
        self._execute(cmd.format(self.STATE_INFO_TABLE,
                                 self.STATE_INFO_TABLE))

        cmd = "PRAGMA user_version = {};"
        self._execute(cmd.format(self.VERSION))

    def load(self):
        retries = 1
        while True:
            assert self.db is None
            assert self.c is None
            assert self.inserts == 0
            empty = not os.path.exists(self.state_file)
            self.db = sqlite3.connect(self.state_file)
            self.c = self.db.cursor()

            # Try loading once to check that the file is indeed a database
            # and reformat it if it is not.
            try:
                self._load(empty=empty)
                return
            except sqlite3.DatabaseError:
                self.c.close()
                self.db.close()
                self.db = None
                self.c = None
                self.inserts = 0
                if retries > 0:
                    os.unlink(self.state_file)
                    retries -= 1
                else:
                    raise

    def dump(self):
        assert self.db is not None

        cmd = "SELECT count from {} WHERE rowid={}"
        self._execute(cmd.format(self.STATE_INFO_TABLE,
                                 self.STATE_INFO_ROW))
        ret = self._fetchall()
        assert len(ret) == 1
        assert len(ret[0]) == 1
        count = ret[0][0] + self.inserts

        if count > self.row_limit:
            msg = "Cleaning up state. This might take a while."
            self.project.logger.warn(msg)

            delete = (count - self.row_limit)
            delete += int(self.row_limit * (self.row_cleanup_quota/100.))
            cmd = "DELETE FROM {} WHERE timestamp IN (" \
                  "SELECT timestamp FROM {} ORDER BY timestamp ASC LIMIT {});"
            self._execute(cmd.format(self.STATE_TABLE,
                                     self.STATE_TABLE,
                                     delete))

            self._execute("VACUUM")

            cmd = "SELECT COUNT(*) FROM {}"

            self._execute(cmd.format(self.STATE_TABLE))
            ret = self._fetchall()
            assert len(ret) == 1
            assert len(ret[0]) == 1
            count = ret[0][0]

        cmd = "UPDATE {} SET count = {} WHERE rowid = {}"
        self._execute(cmd.format(self.STATE_INFO_TABLE,
                                 count,
                                 self.STATE_INFO_ROW))

        self.db.commit()
        self.c.close()
        self.db.close()
        self.db = None
        self.c = None
        self.inserts = 0

    @staticmethod
    def mtime(path):
        mtime = os.path.getmtime(path)
        if os.path.isdir(path):
            for root, dirs, files in os.walk(path):
                for entry in list(dirs) + list(files):
                    p = os.path.join(root, entry)
                    m = os.path.getmtime(p)
                    if m > mtime:
                        mtime = m

        return str(int(nanotime.timestamp(mtime)))

    @staticmethod
    def inode(path):
        return System.inode(path)

    def _do_update(self, path):
        if not os.path.exists(path):
            return (None, None)

        mtime = self.mtime(path)
        inode = self.inode(path)

        cmd = 'SELECT * from {} WHERE inode={}'.format(self.STATE_TABLE,
                                                       inode)

        self._execute(cmd)
        ret = self._fetchall()
        if len(ret) == 0:
            md5, info = self._collect(path)
            cmd = 'INSERT INTO {}(inode, mtime, md5, timestamp) ' \
                  'VALUES ({}, "{}", "{}", "{}")'
            self._execute(cmd.format(self.STATE_TABLE,
                                     inode,
                                     mtime,
                                     md5,
                                     int(nanotime.timestamp(time.time()))))
            self.inserts += 1
        else:
            assert len(ret) == 1
            assert len(ret[0]) == 4
            i, m, md5, timestamp = ret[0]
            assert i == inode
            if mtime != m:
                md5, info = self._collect(path)
                cmd = 'UPDATE {} SET ' \
                      'mtime = "{}", md5 = "{}", timestamp = "{}" ' \
                      'WHERE inode = {}'
                self._execute(cmd.format(self.STATE_TABLE,
                                         mtime,
                                         md5,
                                         int(nanotime.timestamp(time.time())),
                                         inode))
            else:
                info = None
                cmd = 'UPDATE {} SET timestamp = "{}" WHERE inode = {}'
                self._execute(cmd.format(self.STATE_TABLE,
                                         int(nanotime.timestamp(time.time())),
                                         inode))

        return (md5, info)

    def update(self, path):
        return self._do_update(path)[0]

    def update_info(self, path):
        md5, info = self._do_update(path)
        if not info:
            info = self.project.cache.local.load_dir_cache(md5)
        return (md5, info)

    def update_link(self, path):
        if not os.path.exists(path):
            return

        mtime = self.mtime(path)
        inode = self.inode(path)
        relpath = os.path.relpath(path, self.root_dir)

        cmd = 'REPLACE INTO {}(path, inode, mtime) ' \
              'VALUES ("{}", {}, "{}")'.format(self.LINK_STATE_TABLE,
                                               relpath,
                                               inode,
                                               mtime)
        self._execute(cmd)

    def remove_unused_links(self, used):
        unused = []

        self._execute('SELECT * FROM {}'.format(self.LINK_STATE_TABLE))
        for row in self.c:
            p, i, m = row
            path = os.path.join(self.root_dir, p)

            if path in used:
                continue

            if not os.path.exists(path):
                continue

            inode = self.inode(path)
            mtime = self.mtime(path)

            if i == inode and m == mtime:
                Logger.debug('Removing \'{}\' as unused link.'.format(path))
                remove(path)
                unused.append(p)

        for p in unused:
            cmd = 'DELETE FROM {} WHERE path = "{}"'
            self._execute(cmd.format(self.LINK_STATE_TABLE, p))
<commit_msg>state: store uint64 as int64<commit_after>import os
import time
import sqlite3
import nanotime

from dvc.config import Config
from dvc.system import System
from dvc.utils import file_md5, remove
from dvc.exceptions import DvcException
from dvc.logger import Logger

class StateDuplicateError(DvcException):
    pass

class State(object):
    VERSION = 2
    STATE_FILE = 'state'
    STATE_TABLE = 'state'
    STATE_TABLE_LAYOUT = "inode INTEGER PRIMARY KEY, " \
                         "mtime TEXT NOT NULL, " \
                         "md5 TEXT NOT NULL, " \
                         "timestamp TEXT NOT NULL"

    STATE_INFO_TABLE = 'state_info'
    STATE_INFO_TABLE_LAYOUT = 'count INTEGER'
    STATE_INFO_ROW = 1

    LINK_STATE_TABLE = 'link_state'
    LINK_STATE_TABLE_LAYOUT = "path TEXT PRIMARY KEY, " \
                              "inode INTEGER NOT NULL, " \
                              "mtime TEXT NOT NULL"

    STATE_ROW_LIMIT = 10000000
    STATE_ROW_CLEANUP_QUOTA = 50

    MAX_INT = 2**63 - 1
    MAX_UINT = 2**64 - 2

    def __init__(self, project, config):
        self.project = project
        self.dvc_dir = project.dvc_dir
        self.root_dir = project.root_dir

        self.row_limit = 100
        self.row_cleanup_quota = 50

        c = config.get(Config.SECTION_STATE, {})
        self.row_limit = c.get(Config.SECTION_STATE_ROW_LIMIT,
                               self.STATE_ROW_LIMIT)
        self.row_cleanup_quota = c.get(Config.SECTION_STATE_ROW_CLEANUP_QUOTA,
                                       self.STATE_ROW_CLEANUP_QUOTA)

        if not self.dvc_dir:
            self.state_file = None
            return

        self.state_file = os.path.join(self.dvc_dir, self.STATE_FILE)
        self.db = None
        self.c = None
        self.inserts = 0

    def __enter__(self):
        self.load()

    def __exit__(self, type, value, tb):
        self.dump()

    @staticmethod
    def init(project):
        return State(project)

    def _collect(self, path):
        if os.path.isdir(path):
            return self.project.cache.local.collect_dir_cache(path)
        else:
            return (file_md5(path)[0], None)

    def changed(self, path, md5):
        actual = self.update(path)

        if not md5 or not actual:
            return True

        return actual.split('.')[0] != md5.split('.')[0]

    def _execute(self, cmd):
        Logger.debug(cmd)
        return self.c.execute(cmd)

    def _fetchall(self):
        ret = self.c.fetchall()
        Logger.debug("fetched: {}".format(ret))
        return ret

    def _to_sqlite(self, n):
        assert n >= 0
        assert n < self.MAX_UINT
        # NOTE: sqlite stores unit as signed ints, so maximum uint is 2^63-1
        # see http://jakegoulding.com/blog/2011/02/06/sqlite-64-bit-integers/
        if n > self.MAX_INT:
            ret = -(n - self.MAX_INT)
        else:
            ret = n
        assert self._from_sqlite(ret) == n
        return ret

    def _from_sqlite(self, n):
        assert abs(n) <= self.MAX_INT
        if n < 0:
            return abs(n) + self.MAX_INT
        assert n < self.MAX_UINT
        assert n >= 0
        return n

    def _load(self, empty=False):
        from dvc import VERSION

        if not empty:
            cmd = "PRAGMA user_version;"
            self._execute(cmd)
            ret = self._fetchall()
            assert len(ret) == 1
            assert len(ret[0]) == 1
            assert isinstance(ret[0][0], int)
            version = ret[0][0]

            if version > self.VERSION:
                msg = "You are using an old version '{}' of dvc that is " \
                      "using state file version '{}' which is not " \
                      "compatible with the state file version '{}' that " \
                      "is used in this projet. Please upgrade right now!"
                raise DvcException(msg.format(VERSION,
                                              self.VERSION,
                                              version))
            elif version < self.VERSION:
                msg = "State file version '{}' is too old. " \
                      "Reformatting to the current version '{}'."
                self.project.logger.warn(msg.format(version, self.VERSION))
                cmd = "DROP TABLE IF EXISTS {};"
                self._execute(cmd.format(self.STATE_TABLE))
                self._execute(cmd.format(self.STATE_INFO_TABLE))
                self._execute(cmd.format(self.LINK_STATE_TABLE))

        # Check that the state file is indeed a database
        cmd = "CREATE TABLE IF NOT EXISTS {} ({})"
        self._execute(cmd.format(self.STATE_TABLE,
                                 self.STATE_TABLE_LAYOUT))
        self._execute(cmd.format(self.STATE_INFO_TABLE,
                                 self.STATE_INFO_TABLE_LAYOUT))
        self._execute(cmd.format(self.LINK_STATE_TABLE,
                                 self.LINK_STATE_TABLE_LAYOUT))

        cmd = "INSERT OR IGNORE INTO {} (count) SELECT 0 " \
              "WHERE NOT EXISTS (SELECT * FROM {})"
        self._execute(cmd.format(self.STATE_INFO_TABLE,
                                 self.STATE_INFO_TABLE))

        cmd = "PRAGMA user_version = {};"
        self._execute(cmd.format(self.VERSION))

    def load(self):
        retries = 1
        while True:
            assert self.db is None
            assert self.c is None
            assert self.inserts == 0
            empty = not os.path.exists(self.state_file)
            self.db = sqlite3.connect(self.state_file)
            self.c = self.db.cursor()

            # Try loading once to check that the file is indeed a database
            # and reformat it if it is not.
            try:
                self._load(empty=empty)
                return
            except sqlite3.DatabaseError:
                self.c.close()
                self.db.close()
                self.db = None
                self.c = None
                self.inserts = 0
                if retries > 0:
                    os.unlink(self.state_file)
                    retries -= 1
                else:
                    raise

    def dump(self):
        assert self.db is not None

        cmd = "SELECT count from {} WHERE rowid={}"
        self._execute(cmd.format(self.STATE_INFO_TABLE,
                                 self.STATE_INFO_ROW))
        ret = self._fetchall()
        assert len(ret) == 1
        assert len(ret[0]) == 1
        count = self._from_sqlite(ret[0][0]) + self.inserts

        if count > self.row_limit:
            msg = "Cleaning up state. This might take a while."
            self.project.logger.warn(msg)

            delete = (count - self.row_limit)
            delete += int(self.row_limit * (self.row_cleanup_quota/100.))
            cmd = "DELETE FROM {} WHERE timestamp IN (" \
                  "SELECT timestamp FROM {} ORDER BY timestamp ASC LIMIT {});"
            self._execute(cmd.format(self.STATE_TABLE,
                                     self.STATE_TABLE,
                                     delete))

            self._execute("VACUUM")

            cmd = "SELECT COUNT(*) FROM {}"

            self._execute(cmd.format(self.STATE_TABLE))
            ret = self._fetchall()
            assert len(ret) == 1
            assert len(ret[0]) == 1
            count = ret[0][0]

        cmd = "UPDATE {} SET count = {} WHERE rowid = {}"
        self._execute(cmd.format(self.STATE_INFO_TABLE,
                                 self._to_sqlite(count),
                                 self.STATE_INFO_ROW))

        self.db.commit()
        self.c.close()
        self.db.close()
        self.db = None
        self.c = None
        self.inserts = 0

    @staticmethod
    def mtime(path):
        mtime = os.path.getmtime(path)
        if os.path.isdir(path):
            for root, dirs, files in os.walk(path):
                for entry in list(dirs) + list(files):
                    p = os.path.join(root, entry)
                    m = os.path.getmtime(p)
                    if m > mtime:
                        mtime = m

        return str(int(nanotime.timestamp(mtime)))

    @staticmethod
    def inode(path):
        return System.inode(path)

    def _do_update(self, path):
        if not os.path.exists(path):
            return (None, None)

        mtime = self.mtime(path)
        inode = self.inode(path)

        cmd = 'SELECT * from {} WHERE inode={}'.format(self.STATE_TABLE,
                                                       self._to_sqlite(inode))

        self._execute(cmd)
        ret = self._fetchall()
        if len(ret) == 0:
            md5, info = self._collect(path)
            cmd = 'INSERT INTO {}(inode, mtime, md5, timestamp) ' \
                  'VALUES ({}, "{}", "{}", "{}")'
            self._execute(cmd.format(self.STATE_TABLE,
                                     self._to_sqlite(inode),
                                     mtime,
                                     md5,
                                     int(nanotime.timestamp(time.time()))))
            self.inserts += 1
        else:
            assert len(ret) == 1
            assert len(ret[0]) == 4
            i, m, md5, timestamp = ret[0]
            i = self._from_sqlite(i)
            assert i == inode
            if mtime != m:
                md5, info = self._collect(path)
                cmd = 'UPDATE {} SET ' \
                      'mtime = "{}", md5 = "{}", timestamp = "{}" ' \
                      'WHERE inode = {}'
                self._execute(cmd.format(self.STATE_TABLE,
                                         mtime,
                                         md5,
                                         int(nanotime.timestamp(time.time())),
                                         self._to_sqlite(inode)))
            else:
                info = None
                cmd = 'UPDATE {} SET timestamp = "{}" WHERE inode = {}'
                self._execute(cmd.format(self.STATE_TABLE,
                                         int(nanotime.timestamp(time.time())),
                                         self._to_sqlite(inode)))

        return (md5, info)

    def update(self, path):
        return self._do_update(path)[0]

    def update_info(self, path):
        md5, info = self._do_update(path)
        if not info:
            info = self.project.cache.local.load_dir_cache(md5)
        return (md5, info)

    def update_link(self, path):
        if not os.path.exists(path):
            return

        mtime = self.mtime(path)
        inode = self.inode(path)
        relpath = os.path.relpath(path, self.root_dir)

        cmd = 'REPLACE INTO {}(path, inode, mtime) ' \
              'VALUES ("{}", {}, "{}")'.format(self.LINK_STATE_TABLE,
                                               relpath,
                                               self._to_sqlite(inode),
                                               mtime)
        self._execute(cmd)

    def remove_unused_links(self, used):
        unused = []

        self._execute('SELECT * FROM {}'.format(self.LINK_STATE_TABLE))
        for row in self.c:
            p, i, m = row
            i = self._from_sqlite(i)
            path = os.path.join(self.root_dir, p)

            if path in used:
                continue

            if not os.path.exists(path):
                continue

            inode = self.inode(path)
            mtime = self.mtime(path)

            if i == inode and m == mtime:
                Logger.debug('Removing \'{}\' as unused link.'.format(path))
                remove(path)
                unused.append(p)

        for p in unused:
            cmd = 'DELETE FROM {} WHERE path = "{}"'
            self._execute(cmd.format(self.LINK_STATE_TABLE, p))
<|endoftext|>


<commit_before>#!/usr/bin/python
# -*- coding: utf-8 -*-

# Copyright (c) 2015-2016:
#   Frederic Mohier, frederic.mohier@gmail.com
#
# This file is part of (WebUI).
#
# (WebUI) is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# (WebUI) is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with (WebUI).  If not, see <http://www.gnu.org/licenses/>.

"""
    Plugin Livestate
"""

import time
import json

from collections import OrderedDict

from logging import getLogger
from bottle import request, response, redirect

from alignak_webui.objects.item import Item

from alignak_webui.utils.datatable import Datatable

logger = getLogger(__name__)

# Will be populated by the UI with it's own value
webui = None

# Get the same schema as the applications backend and append information for the datatable view
# Use an OrderedDict to create an ordered list of fields
schema = OrderedDict()
# Specific field to include the responsive + button used to display hidden columns on small devices
schema['#'] = {
    'type': 'string',
    'ui': {
        'title': '#',
        # This field is visible (default: False)
        'visible': True,
        # This field is initially hidden (default: False)
        'hidden': False,
        # This field is searchable (default: True)
        'searchable': False,
        # This field is orderable (default: True)
        'orderable': False,
        # search as a regex (else strict value comparing when searching is performed)
        'regex': False,
        # defines the priority for the responsive column hidding (0 is the most important)
        # Default is 10000
        # 'priority': 0,
    }
}
# Specific field to include the commands button
schema['$'] = {
    'type': 'string',
    'ui': {
        'title': '',
        'visible': True,
        'hidden': False,
        'searchable': False,
        'selectable': True,
        'orderable': False,
        'regex': False,
    }
}
schema['type'] = {
    'type': 'string',
    'ui': {
        'title': _('Type'),
        'width': '10px',
        # This field is visible (default: False)
        'visible': True,
        # This field is initially hidden (default: False)
        'hidden': False,
        # This field is searchable (default: True)
        'searchable': True,
        # search as a regex (else strict value comparing when searching is performed)
        'regex': True,
        # This field is orderable (default: True)
        'orderable': True,
        # 'priority': 0,
    },
    'allowed': ["host", "service"]
}
schema['name'] = {
    'type': 'string',
    'ui': {
        'title': _('Element name'),
        'width': '50px',
        'visible': True,
        # 'priority': 0,
    }
}
schema['host'] = {
    'type': 'objectid',
    'ui': {
        'title': _('Host'),
        'width': '10',
        'visible': True,
        'hidden': True,
    },
    'data_relation': {
        'resource': 'host',
        'embeddable': True
    }
}
schema['display_name_host'] = {
    'type': 'string',
    'ui': {
        'title': _('Host display name'),
        'width': '10',
        'visible': True,
        'hidden': True,
    },
}
schema['service'] = {
    'type': 'objectid',
    'ui': {
        'title': _('Service'),
        'width': '10px',
        'visible': True,
        'hidden': True,
    },
    'data_relation': {
        'resource': 'service',
        'embeddable': True
    }
}
schema['display_name_service'] = {
    'type': 'string',
    'ui': {
        'title': _('Host display service'),
        'visible': True,
        'hidden': True,
    },
}
schema['definition_order'] = {
    'type': 'integer',
    'ui': {
        'title': _('Definition order'),
        'visible': True,
        'hidden': True,
        'orderable': False,
    },
}
schema['business_impact'] = {
    'type': 'integer',
    'ui': {
        'title': _('Business impact'),
        'visible': True,
        # 'priority': 1,
    },
    'default': '2',
    'allowed': ["0", "1", "2", "3", "4", "5"]
}
schema['state'] = {
    'type': 'string',
    'ui': {
        'title': _('State'),
        'visible': True,
        'size': 5,
        # 'priority': 0,
    },
    'allowed': ["OK", "WARNING", "CRITICAL", "UNKNOWN", "UP", "DOWN", "UNREACHABLE"]
}
schema['state_type'] = {
    'type': 'string',
    'ui': {
        'title': _('State type'),
        'visible': True,
        # 'priority': 0,
    },
    'allowed': ["HARD", "SOFT"]
}
schema['state_id'] = {
    'type': 'integer',
    'ui': {
        'title': _('State identifier'),
        'visible': True,
        'hidden': True
    },
    'allowed': ['0', '1', '2', '3', '4']
}
schema['acknowledged'] = {
    'type': 'boolean',
    'ui': {
        'title': _('Acknowledged'),
        'visible': True,
        'size': 2,
    },
}
schema['downtime'] = {
    'type': 'boolean',
    'ui': {
        'title': _('In scheduled downtime'),
        'visible': True,
        'width': '20px',
    },
}
schema['last_check'] = {
    'type': 'integer',
    'ui': {
        'title': _('Last check'),
        'visible': True
    },
}
schema['output'] = {
    'type': 'string',
    'ui': {
        'title': _('Check output'),
        'visible': True
    },
}
schema['long_output'] = {
    'type': 'string',
    'ui': {
        'title': _('Check long output'),
        'visible': True
    },
}
schema['perf_data'] = {
    'type': 'string',
    'ui': {
        'title': _('Performance data'),
        'visible': True
    },
}
schema['current_attempt'] = {
    'type': 'integer',
    'ui': {
        'title': _('Current attempt'),
        'visible': True,
        'hidden': True
    },
}
schema['max_attempts'] = {
    'type': 'integer',
    'ui': {
        'title': _('Max attempts'),
        'visible': True,
        'hidden': True
    },
}
schema['next_check'] = {
    'type': 'integer',
    'ui': {
        'title': _('Next check'),
        'visible': True,
        'hidden': True
    },
}
schema['last_state_changed'] = {
    'type': 'integer',
    'ui': {
        'title': _('Last check'),
        'visible': True,
        'hidden': True
    },
}
schema['last_state'] = {
    'type': 'string',
    'ui': {
        'title': _('Last state'),
        'visible': True,
        'hidden': True
    },
    'allowed': ["OK", "WARNING", "CRITICAL", "UNKNOWN", "UP", "DOWN", "UNREACHABLE"]
}
schema['last_state_type'] = {
    'type': 'string',
    'ui': {
        'title': _('Last state type'),
        'visible': True,
        'hidden': True
    },
    'allowed': ["HARD", "SOFT"]
}
schema['latency'] = {
    'type': 'float',
    'ui': {
        'title': _('Latency'),
        'visible': True,
        'hidden': True
    },
}
schema['execution_time'] = {
    'type': 'float',
    'ui': {
        'title': _('Execution time'),
        'visible': True,
        'hidden': True
    },
}

# This to define the global information for the table
schema['ui'] = {
    'type': 'boolean',
    'default': True,

    # UI parameters for the objects
    'ui': {
        'page_title': _('Livestate table (%d items)'),
        'uid': '_id',
        'visible': True,
        'orderable': False,
        'editable': False,
        'selectable': False,
        'searchable': True,
        'responsive': True
    }
}

def get_livestate_table():
    """
    Get the livestate list and transform it as a table
    """
    datamgr = request.environ['beaker.session']['datamanager']

    # Pagination and search
    where = webui.helper.decode_search(request.query.get('search', ''))

    # Get total elements count
    total = datamgr.get_objects_count('livestate', search=where)

    # Build table structure
    dt = Datatable('livestate', datamgr.backend, schema)

    title = dt.title
    if '%d' in title:
        title = title % total

    return {
        'object_type': 'livestate',
        'dt': dt,
        'title': request.query.get('title', title)
    }

def get_livestate_table_data():
    """
    Get the livestate list and provide table data
    """
    datamgr = request.environ['beaker.session']['datamanager']
    dt = Datatable('livestate', datamgr.backend, schema)

    response.status = 200
    response.content_type = 'application/json'
    return dt.table_data()

def get_livestate(element_id):
    """
    Display the element linked to a livestate item
    """
    datamgr = request.environ['beaker.session']['datamanager']

    element = datamgr.get_livestate({'where': {'_id': element_id}})
    if not element:
        return webui.response_invalid_parameters(_('Livestate element does not exist'))

    element = element[0]
    if element.getType() == 'host':
        logger.debug("Livestate: %s %s %s", element, element.host.id, element.__dict__)
        redirect('/host/' + element.host.id)
    else:
        logger.debug("Livestate: %s %s %s", element, element.host.id, element.__dict__)
        redirect('/host/' + element.host.id + '#services')

pages = {
    get_livestate: {
        'name': 'Livestate',
        'route': '/livestate/<element_id>'
    },
    get_livestate_table: {
        'name': 'Livestate table',
        'route': '/livestate_table',
        'view': '_table'
    },

    get_livestate_table_data: {
        'name': 'Livestate table data',
        'route': '/livestate_table_data',
        'method': 'POST'
    },
}
<commit_msg>Livestate typo<commit_after>#!/usr/bin/python
# -*- coding: utf-8 -*-

# Copyright (c) 2015-2016:
#   Frederic Mohier, frederic.mohier@gmail.com
#
# This file is part of (WebUI).
#
# (WebUI) is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# (WebUI) is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with (WebUI).  If not, see <http://www.gnu.org/licenses/>.

"""
    Plugin Livestate
"""

import time
import json

from collections import OrderedDict

from logging import getLogger
from bottle import request, response, redirect

from alignak_webui.objects.item import Item

from alignak_webui.utils.datatable import Datatable

logger = getLogger(__name__)

# Will be populated by the UI with it's own value
webui = None

# Get the same schema as the applications backend and append information for the datatable view
# Use an OrderedDict to create an ordered list of fields
schema = OrderedDict()
# Specific field to include the responsive + button used to display hidden columns on small devices
schema['#'] = {
    'type': 'string',
    'ui': {
        'title': '#',
        # This field is visible (default: False)
        'visible': True,
        # This field is initially hidden (default: False)
        'hidden': False,
        # This field is searchable (default: True)
        'searchable': False,
        # This field is orderable (default: True)
        'orderable': False,
        # search as a regex (else strict value comparing when searching is performed)
        'regex': False,
        # defines the priority for the responsive column hidding (0 is the most important)
        # Default is 10000
        # 'priority': 0,
    }
}
# Specific field to include the commands button
schema['$'] = {
    'type': 'string',
    'ui': {
        'title': '',
        'visible': True,
        'hidden': False,
        'searchable': False,
        'selectable': True,
        'orderable': False,
        'regex': False,
    }
}
schema['type'] = {
    'type': 'string',
    'ui': {
        'title': _('Type'),
        'width': '10px',
        # This field is visible (default: False)
        'visible': True,
        # This field is initially hidden (default: False)
        'hidden': False,
        # This field is searchable (default: True)
        'searchable': True,
        # search as a regex (else strict value comparing when searching is performed)
        'regex': True,
        # This field is orderable (default: True)
        'orderable': True,
        # 'priority': 0,
    },
    'allowed': ["host", "service"]
}
schema['name'] = {
    'type': 'string',
    'ui': {
        'title': _('Element name'),
        'width': '50px',
        'visible': True,
        # 'priority': 0,
    }
}
schema['host'] = {
    'type': 'objectid',
    'ui': {
        'title': _('Host'),
        'width': '10',
        'visible': True,
        'hidden': True,
    },
    'data_relation': {
        'resource': 'host',
        'embeddable': True
    }
}
schema['display_name_host'] = {
    'type': 'string',
    'ui': {
        'title': _('Host display name'),
        'width': '10',
        'visible': True,
        'hidden': True,
    },
}
schema['service'] = {
    'type': 'objectid',
    'ui': {
        'title': _('Service'),
        'width': '10px',
        'visible': True,
        'hidden': True,
    },
    'data_relation': {
        'resource': 'service',
        'embeddable': True
    }
}
schema['display_name_service'] = {
    'type': 'string',
    'ui': {
        'title': _('Host display service'),
        'visible': True,
        'hidden': True,
    },
}
schema['definition_order'] = {
    'type': 'integer',
    'ui': {
        'title': _('Definition order'),
        'visible': True,
        'hidden': True,
        'orderable': False,
    },
}
schema['business_impact'] = {
    'type': 'integer',
    'ui': {
        'title': _('Business impact'),
        'visible': True,
        # 'priority': 1,
    },
    'default': '2',
    'allowed': ["0", "1", "2", "3", "4", "5"]
}
schema['state'] = {
    'type': 'string',
    'ui': {
        'title': _('State'),
        'visible': True,
        'size': 5,
        # 'priority': 0,
    },
    'allowed': ["OK", "WARNING", "CRITICAL", "UNKNOWN", "UP", "DOWN", "UNREACHABLE"]
}
schema['state_type'] = {
    'type': 'string',
    'ui': {
        'title': _('State type'),
        'visible': True,
        # 'priority': 0,
    },
    'allowed': ["HARD", "SOFT"]
}
schema['state_id'] = {
    'type': 'integer',
    'ui': {
        'title': _('State identifier'),
        'visible': True,
        'hidden': True
    },
    'allowed': ['0', '1', '2', '3', '4']
}
schema['acknowledged'] = {
    'type': 'boolean',
    'ui': {
        'title': _('Acknowledged'),
        'visible': True,
        'size': 2,
    },
}
schema['downtime'] = {
    'type': 'boolean',
    'ui': {
        'title': _('In scheduled downtime'),
        'visible': True,
        'width': '20px',
    },
}
schema['last_check'] = {
    'type': 'integer',
    'ui': {
        'title': _('Last check'),
        'visible': True
    },
}
schema['output'] = {
    'type': 'string',
    'ui': {
        'title': _('Check output'),
        'visible': True
    },
}
schema['long_output'] = {
    'type': 'string',
    'ui': {
        'title': _('Check long output'),
        'visible': True
    },
}
schema['perf_data'] = {
    'type': 'string',
    'ui': {
        'title': _('Performance data'),
        'visible': True
    },
}
schema['current_attempt'] = {
    'type': 'integer',
    'ui': {
        'title': _('Current attempt'),
        'visible': True,
        'hidden': True
    },
}
schema['max_attempts'] = {
    'type': 'integer',
    'ui': {
        'title': _('Max attempts'),
        'visible': True,
        'hidden': True
    },
}
schema['next_check'] = {
    'type': 'integer',
    'ui': {
        'title': _('Next check'),
        'visible': True,
        'hidden': True
    },
}
schema['last_state_changed'] = {
    'type': 'integer',
    'ui': {
        'title': _('Last check'),
        'visible': True,
        'hidden': True
    },
}
schema['last_state'] = {
    'type': 'string',
    'ui': {
        'title': _('Last state'),
        'visible': True,
        'hidden': True
    },
    'allowed': ["OK", "WARNING", "CRITICAL", "UNKNOWN", "UP", "DOWN", "UNREACHABLE"]
}
schema['last_state_type'] = {
    'type': 'string',
    'ui': {
        'title': _('Last state type'),
        'visible': True,
        'hidden': True
    },
    'allowed': ["HARD", "SOFT"]
}
schema['latency'] = {
    'type': 'float',
    'ui': {
        'title': _('Latency'),
        'visible': True,
        'hidden': True
    },
}
schema['execution_time'] = {
    'type': 'float',
    'ui': {
        'title': _('Execution time'),
        'visible': True,
        'hidden': True
    },
}

# This to define the global information for the table
schema['ui'] = {
    'type': 'boolean',
    'default': True,

    # UI parameters for the objects
    'ui': {
        'page_title': _('Livestate table (%d items)'),
        'uid': '_id',
        'visible': True,
        'orderable': False,
        'editable': False,
        'selectable': False,
        'searchable': True,
        'responsive': True
    }
}

def get_livestate_table():
    """
    Get the livestate list and transform it as a table
    """
    datamgr = request.environ['beaker.session']['datamanager']

    # Pagination and search
    where = webui.helper.decode_search(request.query.get('search', ''))

    # Get total elements count
    total = datamgr.get_objects_count('livestate', search=where)

    # Build table structure
    dt = Datatable('livestate', datamgr.backend, schema)

    title = dt.title
    if '%d' in title:
        title = title % total

    return {
        'object_type': 'livestate',
        'dt': dt,
        'title': request.query.get('title', title)
    }

def get_livestate_table_data():
    """
    Get the livestate list and provide table data
    """
    datamgr = request.environ['beaker.session']['datamanager']
    dt = Datatable('livestate', datamgr.backend, schema)

    response.status = 200
    response.content_type = 'application/json'
    return dt.table_data()

def get_livestate(livestate_id):
    """
    Display the element linked to a livestate item
    """
    datamgr = request.environ['beaker.session']['datamanager']

    livestate = datamgr.get_livestate({'where': {'_id': livestate_id}})
    if not livestate:
        return webui.response_invalid_parameters(_('Livestate element does not exist'))

    livestate = livestate[0]
    if livestate.getType() == 'host':
        logger.debug("Livestate: %s %s %s", livestate, livestate.host.id, livestate.__dict__)
        redirect('/host/' + livestate.host.id)
    else:
        logger.debug("Livestate: %s %s %s", livestate, livestate.host.id, livestate.__dict__)
        redirect('/host/' + livestate.host.id + '#services')

pages = {
    get_livestate: {
        'name': 'Livestate',
        'route': '/livestate/<livestate_id>'
    },
    get_livestate_table: {
        'name': 'Livestate table',
        'route': '/livestate_table',
        'view': '_table'
    },

    get_livestate_table_data: {
        'name': 'Livestate table data',
        'route': '/livestate_table_data',
        'method': 'POST'
    },
}
<|endoftext|>


<commit_before># -*- coding: utf8 -*-

# Inspired by https://github.com/openai/gym/blob/master/gym/envs/registration.py
# When making an environment, we first look if we registered a version of it ourselves.
# Else, we make just make one using the Environment class.
import numpy as np

import gym
from misc.utils import AtariRescale42x42
from universe.wrappers import Unvectorize, Vectorize

class EnvSpec(gym.envs.registration.EnvSpec):
    """
    Modified version of `gym.envs.registration.EnvSpec`
    to allow env initalization with different parameters.
    """

    def make(self, **kwargs):
        """Instantiates an instance of the environment with appropriate kwargs"""
        if self._entry_point is None:
            raise gym.error.Error('Attempting to make deprecated env {}. (HINT: is there a newer registered version of this env?)'.format(self.id))

        elif callable(self._entry_point):
            env = self._entry_point()
        else:
            cls = gym.envs.registration.load(self._entry_point)
            all_kwargs = self._kwargs
            all_kwargs.update(kwargs)
            env = cls(**all_kwargs)

        # Make the enviroment aware of which spec it came from.
        env.unwrapped._spec = self

        return env

def make(id, **kwargs):
    spec = gym.envs.registry.spec(id)
    env = spec.make(**kwargs)
    def to_dict():
        return {"id": id}
    if "to_dict" not in dir(env):
        env.to_dict = to_dict
    if "atari.atari_env" in env.unwrapped.__module__:
        env = Vectorize(env)
        env = AtariRescale42x42(env)
        env = Unvectorize(env)
    if (env.spec.timestep_limit is not None) and not spec.tags.get('vnc'):
        from gym.wrappers.time_limit import TimeLimit
        env = TimeLimit(env,
                        max_episode_steps=env.spec.max_episode_steps,
                        max_episode_seconds=env.spec.max_episode_seconds)
        env.to_dict = env.env.to_dict
    return env

def make_environments(descriptions):
    """Make environments using a list of descriptions."""
    return [make(**d) for d in descriptions]

def make_random_environments(id, n_envs):
    """Make n_envs random environments of the env_name class."""
    spec = gym.envs.registry.spec(id)
    cls = gym.envs.registration.load(spec._entry_point)
    envs = []
    for _ in range(n_envs):
        args = {"id": id}
        for p in cls.changeable_parameters:
            if p["type"] == "range":
                args[p["name"]] = np.random.uniform(p["low"], p["high"])  # Assume for now only range parameters are used
        envs.append(make(**args))
    return envs
<commit_msg>Second experiment running fix<commit_after># -*- coding: utf8 -*-

# Inspired by https://github.com/openai/gym/blob/master/gym/envs/registration.py
# When making an environment, we first look if we registered a version of it ourselves.
# Else, we make just make one using the Environment class.
import numpy as np

import gym
from misc.utils import AtariRescale42x42
from universe.wrappers import Unvectorize, Vectorize

class EnvSpec(gym.envs.registration.EnvSpec):
    """
    Modified version of `gym.envs.registration.EnvSpec`
    to allow env initalization with different parameters.
    """

    def make(self, **kwargs):
        """Instantiates an instance of the environment with appropriate kwargs"""
        if self._entry_point is None:
            raise gym.error.Error('Attempting to make deprecated env {}. (HINT: is there a newer registered version of this env?)'.format(self.id))

        elif callable(self._entry_point):
            env = self._entry_point()
        else:
            cls = gym.envs.registration.load(self._entry_point)
            all_kwargs = self._kwargs
            all_kwargs.update(kwargs)
            env = cls(**all_kwargs)

        # Make the enviroment aware of which spec it came from.
        env.unwrapped._spec = self

        return env

def make(id, **kwargs):
    spec = gym.envs.registry.spec(id)
    env = spec.make(**kwargs)
    def to_dict():
        return {"id": id}
    if "to_dict" not in dir(env):
        env.to_dict = to_dict
    if "atari.atari_env" in env.unwrapped.__module__:
        to_dict = env.to_dict
        env = Vectorize(env)
        env = AtariRescale42x42(env)
        env = Unvectorize(env)
        env.to_dict = to_dict
    if (env.spec.timestep_limit is not None) and not spec.tags.get('vnc'):
        from gym.wrappers.time_limit import TimeLimit
        env = TimeLimit(env,
                        max_episode_steps=env.spec.max_episode_steps,
                        max_episode_seconds=env.spec.max_episode_seconds)
        env.to_dict = env.env.to_dict
    return env

def make_environments(descriptions):
    """Make environments using a list of descriptions."""
    return [make(**d) for d in descriptions]

def make_random_environments(id, n_envs):
    """Make n_envs random environments of the env_name class."""
    spec = gym.envs.registry.spec(id)
    cls = gym.envs.registration.load(spec._entry_point)
    envs = []
    for _ in range(n_envs):
        args = {"id": id}
        for p in cls.changeable_parameters:
            if p["type"] == "range":
                args[p["name"]] = np.random.uniform(p["low"], p["high"])  # Assume for now only range parameters are used
        envs.append(make(**args))
    return envs
<|endoftext|>


<commit_before>"""Operations regarding miscellaneous files."""

from __future__ import absolute_import, division, print_function
from contextlib import contextmanager
import logging
import re
import shutil
import tempfile
import zipfile

import numpy as np

# TODO: Remove this in favor of .types.
# With older Python version, pathlib2 might be preferred.
try:
    from pathlib2 import Path, PurePath
except ImportError:
    from pathlib import Path, PurePath

import dwi.asciifile
import dwi.dicomfile
import dwi.hdf5
# from .types import Path, PurePath

log = logging.getLogger(__name__)
COMMENT_PREFIX = '#'

@contextmanager
def temp_dir():
    """A temporary directory context that deletes it afterwards."""
    tmpdir = tempfile.mkdtemp()
    try:
        yield tmpdir
    finally:
        shutil.rmtree(tmpdir)

@contextmanager
def read_archive(archive, max_size=None):
    """Extract a ZIP archive into a temporary directory, use as a context."""
    with zipfile.ZipFile(archive, 'r') as a:
        infos = a.infolist()
        total_size = sum(x.file_size for x in infos)
        if max_size is not None and total_size > max_size:
            raise ValueError('Archive content is too big', archive)
        with temp_dir() as tmpdir:
            # Extract one by one because the docs warn against extractall().
            log.debug('Extracting %s to %s', archive, tmpdir)
            for info in infos:
                a.extract(info, tmpdir)
            yield tmpdir

def sanitize_line(line):
    """Clean up an input line."""
    return line.split(COMMENT_PREFIX, 1)[0].strip()

def valid_lines(path):
    """Read and yield lines that are neither empty nor comments."""
    with Path(path).open(mode='rU') as fp:
        for line in filter(None, (sanitize_line(x) for x in fp)):
            yield line

def ensure_dir(path):
    """Ensure existence of the file's parent directory."""
    Path(path).parent.mkdir(parents=True, exist_ok=True)

@contextmanager
def mapped(shape, dtype, fill_value=None):
    """Create an array as a memory-mapped temporary file on disk."""
    with tempfile.TemporaryFile() as fp:
        a = np.memmap(fp, dtype=dtype, mode='w+', shape=shape)
        if fill_value is not None:
            a.fill(fill_value)
        yield a

def parse_patient(line, include_lines=False):
    """Parse a line in patient list file.
    Format: num name scan1,scan2,... score [location]
    """
    regexp = r"""
        (?P<num>\d+) \s+
        (?P<name>\w+) \s+
        (?P<scans>[\w,]+) \s+
        (?P<score>\d\+\d(\+\d)?) \s* (?P<location>\w+)? \s*
        ((?P<score2>\d\+\d(\+\d)?) \s+ (?P<location2>\w+))? \s*
        ((?P<score3>\d\+\d(\+\d)?) \s+ (?P<location3>\w+))?
        """
    p = re.compile(regexp, flags=re.VERBOSE)
    m = p.match(line)
    if m is None:
        raise ValueError('Invalid line in patients file: %s', line)
    num = m.group('num')
    name = m.group('name')
    scans = sorted(m.group('scans').lower().split(','))
    les = [dwi.Lesion(0, m.group('score'), 'xx')]
    if m.group('location'):
        # New-style, multi-lesion file.
        les = []
        les.append(dwi.Lesion(0, m.group('score'), m.group('location')))
        if m.group('score2'):
            les.append(dwi.Lesion(1, m.group('score2'), m.group('location2')))
        if m.group('score3'):
            les.append(dwi.Lesion(2, m.group('score3'), m.group('location3')))
    patient = dwi.Patient(num, name, scans, les)
    if include_lines:
        patient.line = line
    return patient

def read_patients_file(path, include_lines=False):
    """Read a list of patients."""
    return sorted(parse_patient(x, include_lines=include_lines) for x in
                  valid_lines(path))

def parse_sample(line):
    """Parse a line in sample list file.
    Format: num name scan1,scan2,...
    """
    p = re.compile(r'(\d+)\s+(\w+)\s+([\w,]+)')
    m = p.match(line)
    if m is None:
        raise ValueError('Invalid line in samplelist file: %s', line)
    case, name, scans = m.groups()
    case = int(case)
    name = name.lower()
    scans = tuple(sorted(scans.lower().split(',')))
    return dict(case=case, name=name, scans=scans)

def read_sample_list(path):
    """Read a list of samples from file."""
    return [parse_sample(x) for x in valid_lines(path)]

def read_subregion_file(filename):
    """Read a subregion definition from file.

    It's formatted as one voxel index per line, zero-based, in order of
    y_first, y_last, x_first, x_last, z_first, z_last. The "last" ones need +1
    to get Python-like start:stop indices. They are returned as (z_start,
    z_stop, y_start, y_stop, x_start, x_stop).
    """
    entries = []
    for line in valid_lines(filename):
        entries.append(int(float(line)))
    if len(entries) != 6:
        raise Exception('Invalid subregion file: %s' % filename)
    entries = entries[4:6] + entries[0:4]  # Move z indices to front.
    # Add one to "last" indices get Python-like start:stop indices.
    entries[1] += 1
    entries[3] += 1
    entries[5] += 1
    return tuple(entries)

def write_comment(f, text):
    """Write zero or more lines to file with comment prefix."""
    for line in text.splitlines():
        f.write('{p} {s}\n'.format(p=COMMENT_PREFIX, s=line))

def write_subregion_file(filename, win, comment=''):
    """Write a subregion definition to file.

    It's formatted as one voxel index per line, zero-based, in order of
    y_first, y_last, x_first, x_last, z_first, z_last.
    """
    if len(win) != 6:
        raise Exception('Invalid subregion: %s' % win)
    entries = [win[2], win[3]-1, win[4], win[5]-1, win[0], win[1]-1]
    with open(filename, 'w') as f:
        write_comment(f, comment)
        for entry in entries:
            f.write('%i\n' % entry)

def guess_format(path):
    """Guess file format identifier from it's suffix."""
    return PurePath(path).suffix[1:]

def write_pmap(filename, pmap, attrs, fmt=None):
    """Write parametric map file either as HDF5 or ASCII."""
    pmap = np.asanyarray(pmap)
    if pmap.ndim < 2:
        raise Exception('Not enough dimensions: {}'.format(pmap.shape))
    if 'parameters' not in attrs:
        attrs['parameters'] = [str(i) for i in range(pmap.shape[-1])]
    if 'shape' not in attrs:
        attrs['shape'] = pmap.shape
    if 'dtype' not in attrs:
        attrs['dtype'] = str(pmap.dtype)
    if pmap.shape[-1] != len(attrs['parameters']):
        raise Exception('Number of values and parameters mismatch')
    assert all(dwi.util.isstring(x) for x in
               attrs['parameters']), attrs['parameters']
    if fmt is None:
        fmt = guess_format(filename)
    if fmt == 'h5':
        dwi.hdf5.write_hdf5(filename, pmap, attrs)
    elif fmt == 'txt':
        pmap = pmap.reshape((-1, pmap.shape[-1]))  # Can't keep shape.
        dwi.asciifile.write_ascii_file(filename, pmap, None, attrs=attrs)
    else:
        raise Exception('Unknown format: {}'.format(fmt))

def asindices(iterable, lst):
    """Replace items in iterable with their corresponding indices in list, or
    convert them to integers. E.g. asindices(['o', '22'], 'foobar') => 1, 22.
    """
    for item in iterable:
        try:
            yield lst.index(item)  # Yield corresponding index.
        except ValueError:
            yield int(item)  # Not found, convert to integer.

def pick_params(pmap, attrs, params):
    """Select a subset of parameters by their indices or names."""
    params = list(asindices(params, attrs['parameters']))
    pmap = pmap[..., params]
    if 'bset' in attrs and len(attrs['bset']) == len(attrs['parameters']):
        attrs['bset'] = [attrs['bset'][x] for x in params]
    if 'echotimes' in attrs and (len(attrs['echotimes']) ==
                                 len(attrs['parameters'])):
        attrs['echotimes'] = [attrs['echotimes'][x] for x in params]
    attrs['parameters'] = [attrs['parameters'][x] for x in params]
    return pmap, attrs

def read_pmap(path, ondisk=False, fmt=None, params=None, dtype=None):
    """Read a parametric map.

    With parameter ondisk it will not be read into memory. Parameter params
    tells which parameter indices should be included.
    """
    if fmt is None:
        fmt = guess_format(path)
    if fmt == 'h5':
        pmap, attrs = dwi.hdf5.read_hdf5(path, ondisk=ondisk)
    elif fmt == 'txt':
        attrs, pmap = dwi.asciifile.read_ascii_file(path)
        if 'parameters' in attrs:
            attrs['parameters'] = attrs['parameters'].split()
    elif fmt == 'zip':
        with read_archive(path) as tempdir:
            return read_pmap(tempdir, ondisk=ondisk, fmt=None, params=params,
                             dtype=dtype)
    else:
        # No extension, assume it's a DICOM directory.
        d = dwi.dicomfile.read_dir(path)
        pmap = d.pop('image')
        attrs = dict(d)
    if 'parameters' not in attrs:
        attrs['parameters'] = range(pmap.shape[-1])
    attrs['parameters'] = [str(x) for x in attrs['parameters']]
    if params is not None:
        pmap, attrs = pick_params(pmap, attrs, params)
    if dtype is not None:
        pmap = pmap.astype(dtype)
    log.debug('Read %s, %s, %s', path, pmap.shape, pmap.dtype)
    return pmap, attrs

def check_spacing(attrs, expected_voxel_spacing, n_dec):
    """Check if voxel spacing is as expected."""
    vs = [round(x, n_dec) for x in attrs['voxel_spacing']]
    evs = [round(x, n_dec) for x in expected_voxel_spacing]
    if vs != evs:
        raise ValueError('Expected voxel spacing {}, got {}'.format(evs, vs))

def check_container(mask, container, allowed_outside):
    """Check if mask stays nicely within another."""
    portion_outside_container = (np.count_nonzero(mask[~container]) /
                                 np.count_nonzero(mask))
    if portion_outside_container > allowed_outside:
        s = 'Portion of selected voxels outside container is {:%}'
        raise ValueError(s.format(portion_outside_container))

def read_mask(path, expected_voxel_spacing=None, n_dec=3, container=None,
              allowed_outside=0.2):
    """Read pmap as a mask.

    Optionally expect voxel spacing to match up to a certain number of
    decimals. The optional parameter allowed_outside indicates how much of the
    smaller mask (lesion) volume may be outside of a larger container mask
    (prostate) without an error being raised.
    """
    mask, attrs = read_pmap(path)
    mask = mask[..., 0].astype(np.bool)
    if expected_voxel_spacing is not None:
        check_spacing(attrs, expected_voxel_spacing, n_dec)
    if container is not None:
        check_container(mask, container, allowed_outside)
    return mask
<commit_msg>import style<commit_after>"""Operations regarding miscellaneous files."""

from __future__ import absolute_import, division, print_function
from contextlib import contextmanager
import logging
import re
import shutil
import tempfile
import zipfile

import numpy as np

from . import asciifile, dicomfile, hdf5, util
from .types import Lesion, Patient
from .types import Path, PurePath

log = logging.getLogger(__name__)
COMMENT_PREFIX = '#'

@contextmanager
def temp_dir():
    """A temporary directory context that deletes it afterwards."""
    tmpdir = tempfile.mkdtemp()
    try:
        yield tmpdir
    finally:
        shutil.rmtree(tmpdir)

@contextmanager
def read_archive(archive, max_size=None):
    """Extract a ZIP archive into a temporary directory, use as a context."""
    with zipfile.ZipFile(archive, 'r') as a:
        infos = a.infolist()
        total_size = sum(x.file_size for x in infos)
        if max_size is not None and total_size > max_size:
            raise ValueError('Archive content is too big', archive)
        with temp_dir() as tmpdir:
            # Extract one by one because the docs warn against extractall().
            log.debug('Extracting %s to %s', archive, tmpdir)
            for info in infos:
                a.extract(info, tmpdir)
            yield tmpdir

def sanitize_line(line):
    """Clean up an input line."""
    return line.split(COMMENT_PREFIX, 1)[0].strip()

def valid_lines(path):
    """Read and yield lines that are neither empty nor comments."""
    with Path(path).open(mode='rU') as fp:
        for line in filter(None, (sanitize_line(x) for x in fp)):
            yield line

def ensure_dir(path):
    """Ensure existence of the file's parent directory."""
    Path(path).parent.mkdir(parents=True, exist_ok=True)

@contextmanager
def mapped(shape, dtype, fill_value=None):
    """Create an array as a memory-mapped temporary file on disk."""
    with tempfile.TemporaryFile() as fp:
        a = np.memmap(fp, dtype=dtype, mode='w+', shape=shape)
        if fill_value is not None:
            a.fill(fill_value)
        yield a

def parse_patient(line, include_lines=False):
    """Parse a line in patient list file.
    Format: num name scan1,scan2,... score [location]
    """
    regexp = r"""
        (?P<num>\d+) \s+
        (?P<name>\w+) \s+
        (?P<scans>[\w,]+) \s+
        (?P<score>\d\+\d(\+\d)?) \s* (?P<location>\w+)? \s*
        ((?P<score2>\d\+\d(\+\d)?) \s+ (?P<location2>\w+))? \s*
        ((?P<score3>\d\+\d(\+\d)?) \s+ (?P<location3>\w+))?
        """
    p = re.compile(regexp, flags=re.VERBOSE)
    m = p.match(line)
    if m is None:
        raise ValueError('Invalid line in patients file: %s', line)
    num = m.group('num')
    name = m.group('name')
    scans = sorted(m.group('scans').lower().split(','))
    les = [Lesion(0, m.group('score'), 'xx')]
    if m.group('location'):
        # New-style, multi-lesion file.
        les = []
        les.append(Lesion(0, m.group('score'), m.group('location')))
        if m.group('score2'):
            les.append(Lesion(1, m.group('score2'), m.group('location2')))
        if m.group('score3'):
            les.append(Lesion(2, m.group('score3'), m.group('location3')))
    patient = Patient(num, name, scans, les)
    if include_lines:
        patient.line = line
    return patient

def read_patients_file(path, include_lines=False):
    """Read a list of patients."""
    return sorted(parse_patient(x, include_lines=include_lines) for x in
                  valid_lines(path))

def parse_sample(line):
    """Parse a line in sample list file.
    Format: num name scan1,scan2,...
    """
    p = re.compile(r'(\d+)\s+(\w+)\s+([\w,]+)')
    m = p.match(line)
    if m is None:
        raise ValueError('Invalid line in samplelist file: %s', line)
    case, name, scans = m.groups()
    case = int(case)
    name = name.lower()
    scans = tuple(sorted(scans.lower().split(',')))
    return dict(case=case, name=name, scans=scans)

def read_sample_list(path):
    """Read a list of samples from file."""
    return [parse_sample(x) for x in valid_lines(path)]

def read_subregion_file(filename):
    """Read a subregion definition from file.

    It's formatted as one voxel index per line, zero-based, in order of
    y_first, y_last, x_first, x_last, z_first, z_last. The "last" ones need +1
    to get Python-like start:stop indices. They are returned as (z_start,
    z_stop, y_start, y_stop, x_start, x_stop).
    """
    entries = []
    for line in valid_lines(filename):
        entries.append(int(float(line)))
    if len(entries) != 6:
        raise Exception('Invalid subregion file: %s' % filename)
    entries = entries[4:6] + entries[0:4]  # Move z indices to front.
    # Add one to "last" indices get Python-like start:stop indices.
    entries[1] += 1
    entries[3] += 1
    entries[5] += 1
    return tuple(entries)

def write_comment(f, text):
    """Write zero or more lines to file with comment prefix."""
    for line in text.splitlines():
        f.write('{p} {s}\n'.format(p=COMMENT_PREFIX, s=line))

def write_subregion_file(filename, win, comment=''):
    """Write a subregion definition to file.

    It's formatted as one voxel index per line, zero-based, in order of
    y_first, y_last, x_first, x_last, z_first, z_last.
    """
    if len(win) != 6:
        raise Exception('Invalid subregion: %s' % win)
    entries = [win[2], win[3]-1, win[4], win[5]-1, win[0], win[1]-1]
    with open(filename, 'w') as f:
        write_comment(f, comment)
        for entry in entries:
            f.write('%i\n' % entry)

def guess_format(path):
    """Guess file format identifier from it's suffix."""
    return PurePath(path).suffix[1:]

def write_pmap(filename, pmap, attrs, fmt=None):
    """Write parametric map file either as HDF5 or ASCII."""
    pmap = np.asanyarray(pmap)
    if pmap.ndim < 2:
        raise Exception('Not enough dimensions: {}'.format(pmap.shape))
    if 'parameters' not in attrs:
        attrs['parameters'] = [str(i) for i in range(pmap.shape[-1])]
    if 'shape' not in attrs:
        attrs['shape'] = pmap.shape
    if 'dtype' not in attrs:
        attrs['dtype'] = str(pmap.dtype)
    if pmap.shape[-1] != len(attrs['parameters']):
        raise Exception('Number of values and parameters mismatch')
    assert all(util.isstring(x) for x in
               attrs['parameters']), attrs['parameters']
    if fmt is None:
        fmt = guess_format(filename)
    if fmt == 'h5':
        hdf5.write_hdf5(filename, pmap, attrs)
    elif fmt == 'txt':
        pmap = pmap.reshape((-1, pmap.shape[-1]))  # Can't keep shape.
        asciifile.write_ascii_file(filename, pmap, None, attrs=attrs)
    else:
        raise Exception('Unknown format: {}'.format(fmt))

def asindices(iterable, lst):
    """Replace items in iterable with their corresponding indices in list, or
    convert them to integers. E.g. asindices(['o', '22'], 'foobar') => 1, 22.
    """
    for item in iterable:
        try:
            yield lst.index(item)  # Yield corresponding index.
        except ValueError:
            yield int(item)  # Not found, convert to integer.

def pick_params(pmap, attrs, params):
    """Select a subset of parameters by their indices or names."""
    params = list(asindices(params, attrs['parameters']))
    pmap = pmap[..., params]
    if 'bset' in attrs and len(attrs['bset']) == len(attrs['parameters']):
        attrs['bset'] = [attrs['bset'][x] for x in params]
    if 'echotimes' in attrs and (len(attrs['echotimes']) ==
                                 len(attrs['parameters'])):
        attrs['echotimes'] = [attrs['echotimes'][x] for x in params]
    attrs['parameters'] = [attrs['parameters'][x] for x in params]
    return pmap, attrs

def read_pmap(path, ondisk=False, fmt=None, params=None, dtype=None):
    """Read a parametric map.

    With parameter ondisk it will not be read into memory. Parameter params
    tells which parameter indices should be included.
    """
    if fmt is None:
        fmt = guess_format(path)
    if fmt == 'h5':
        pmap, attrs = hdf5.read_hdf5(path, ondisk=ondisk)
    elif fmt == 'txt':
        attrs, pmap = asciifile.read_ascii_file(path)
        if 'parameters' in attrs:
            attrs['parameters'] = attrs['parameters'].split()
    elif fmt == 'zip':
        with read_archive(path) as tempdir:
            return read_pmap(tempdir, ondisk=ondisk, fmt=None, params=params,
                             dtype=dtype)
    else:
        # No extension, assume it's a DICOM directory.
        d = dicomfile.read_dir(path)
        pmap = d.pop('image')
        attrs = dict(d)
    if 'parameters' not in attrs:
        attrs['parameters'] = range(pmap.shape[-1])
    attrs['parameters'] = [str(x) for x in attrs['parameters']]
    if params is not None:
        pmap, attrs = pick_params(pmap, attrs, params)
    if dtype is not None:
        pmap = pmap.astype(dtype)
    log.debug('Read %s, %s, %s', path, pmap.shape, pmap.dtype)
    return pmap, attrs

def check_spacing(attrs, expected_voxel_spacing, n_dec):
    """Check if voxel spacing is as expected."""
    vs = [round(x, n_dec) for x in attrs['voxel_spacing']]
    evs = [round(x, n_dec) for x in expected_voxel_spacing]
    if vs != evs:
        raise ValueError('Expected voxel spacing {}, got {}'.format(evs, vs))

def check_container(mask, container, allowed_outside):
    """Check if mask stays nicely within another."""
    portion_outside_container = (np.count_nonzero(mask[~container]) /
                                 np.count_nonzero(mask))
    if portion_outside_container > allowed_outside:
        s = 'Portion of selected voxels outside container is {:%}'
        raise ValueError(s.format(portion_outside_container))

def read_mask(path, expected_voxel_spacing=None, n_dec=3, container=None,
              allowed_outside=0.2):
    """Read pmap as a mask.

    Optionally expect voxel spacing to match up to a certain number of
    decimals. The optional parameter allowed_outside indicates how much of the
    smaller mask (lesion) volume may be outside of a larger container mask
    (prostate) without an error being raised.
    """
    mask, attrs = read_pmap(path)
    mask = mask[..., 0].astype(np.bool)
    if expected_voxel_spacing is not None:
        check_spacing(attrs, expected_voxel_spacing, n_dec)
    if container is not None:
        check_container(mask, container, allowed_outside)
    return mask
<|endoftext|>