<commit_before># -*- coding: utf-8 -*-
'''
    salt.log
    ~~~~~~~~

    This is where Salt's logging gets set up.

    :copyright: 2011-2012 :email:`Pedro Algarvio (pedro@algarvio.me)`
    :license: Apache 2.0, see LICENSE for more details.
'''

# Import python libs
import os
import re
import sys
import socket
import urlparse
import logging
import logging.handlers

TRACE = logging.TRACE = 5
GARBAGE = logging.GARBAGE = 1

LOG_LEVELS = {
    'all': logging.NOTSET,
    'debug': logging.DEBUG,
    'error': logging.ERROR,
    'garbage': GARBAGE,
    'info': logging.INFO,
    'quiet': 1000,
    'trace': TRACE,
    'warning': logging.WARNING,
}

# Make a list of log level names sorted by log level
SORTED_LEVEL_NAMES = [
    l[0] for l in sorted(LOG_LEVELS.iteritems(), key=lambda x: x[1])
]

# Store an instance of the current logging logger class
LOGGING_LOGGER_CLASS = logging.getLoggerClass()

MODNAME_PATTERN = re.compile(r'(?P<name>%%\(name\)(?:\-(?P<digits>[\d]+))?s)')

__CONSOLE_CONFIGURED = False
__LOGFILE_CONFIGURED = False

def is_console_configured():
    return __CONSOLE_CONFIGURED

def is_logfile_configured():
    return __LOGFILE_CONFIGURED

def is_logging_configured():
    return __CONSOLE_CONFIGURED or __LOGFILE_CONFIGURED

if sys.version_info < (2, 7):
    # Since the NullHandler is only available on python >= 2.7, here's a copy
    class NullHandler(logging.Handler):
        """ This is 1 to 1 copy of python's 2.7 NullHandler"""
        def handle(self, record):
            pass

        def emit(self, record):
            pass

        def createLock(self):  # pylint: disable-msg=C0103
            self.lock = None

    logging.NullHandler = NullHandler

# Store a reference to the null logging handler
LOGGING_NULL_HANDLER = logging.NullHandler()

class LoggingTraceMixIn(object):
    '''
    Simple mix-in class to add a trace method to python's logging.
    '''

    def trace(self, msg, *args, **kwargs):
        self.log(TRACE, msg, *args, **kwargs)

class LoggingGarbageMixIn(object):
    '''
    Simple mix-in class to add a garbage method to python's logging.
    '''

    def garbage(self, msg, *args, **kwargs):
        self.log(GARBAGE, msg, *args, **kwargs)

class LoggingMixInMeta(type):
    '''
    This class is called whenever a new instance of ``SaltLoggingClass`` is
    created.

    What this class does is check if any of the bases have a `trace()` or a
    `garbage()` method defined, if they don't we add the respective mix-ins to
    the bases.
    '''
    def __new__(mcs, name, bases, attrs):
        include_trace = include_garbage = True
        bases = list(bases)
        if name == 'SaltLoggingClass':
            for base in bases:
                if hasattr(base, 'trace'):
                    include_trace = False
                if hasattr(base, 'garbage'):
                    include_garbage = False
        if include_trace:
            bases.append(LoggingTraceMixIn)
        if include_garbage:
            bases.append(LoggingGarbageMixIn)
        return super(LoggingMixInMeta, mcs).__new__(
            mcs, name, tuple(bases), attrs
        )

class SaltLoggingClass(LOGGING_LOGGER_CLASS):
    __metaclass__ = LoggingMixInMeta

    def __new__(mcs, logger_name):
        '''
        We override `__new__` in our logging logger class in order to provide
        some additional features like expand the module name padding if length
        is being used, and also some Unicode fixes.

        This code overhead will only be executed when the class is
        instantiated, ie:

            logging.getLogger(__name__)

        '''
        instance = super(SaltLoggingClass, mcs).__new__(mcs)

        try:
            max_logger_length = len(max(
                logging.Logger.manager.loggerDict.keys(), key=len
            ))
            for handler in logging.getLogger().handlers:
                if handler is LOGGING_NULL_HANDLER:
                    continue

                if not handler.lock:
                    handler.createLock()
                handler.acquire()

                formatter = handler.formatter
                fmt = formatter._fmt.replace('%', '%%')

                match = MODNAME_PATTERN.search(fmt)
                if not match:
                    # Not matched. Release handler and return.
                    handler.release()
                    return instance

                if 'digits' not in match.groupdict():
                    # No digits group. Release handler and return.
                    handler.release()
                    return instance

                digits = match.group('digits')
                if not digits or not (digits and digits.isdigit()):
                    # No valid digits. Release handler and return.
                    handler.release()
                    return instance

                if int(digits) < max_logger_length:
                    # Formatter digits value is lower than current max, update.
                    fmt = fmt.replace(match.group('name'), '%%(name)-%ds')
                    formatter = logging.Formatter(
                        fmt % max_logger_length,
                        datefmt=formatter.datefmt
                    )
                    handler.setFormatter(formatter)
                handler.release()
        except ValueError:
            # There are no registered loggers yet
            pass
        return instance

    # pylint: disable-msg=C0103
    def makeRecord(self, name, level, fn, lno, msg, args, exc_info, func=None,
                   extra=None):
        # Let's try to make every logging message unicode
        if isinstance(msg, basestring) and not isinstance(msg, unicode):
            try:
                return LOGGING_LOGGER_CLASS.makeRecord(
                    self, name, level, fn, lno,
                    msg.decode('utf-8', 'replace'),
                    args, exc_info, func, extra
                )
            except UnicodeDecodeError:
                return LOGGING_LOGGER_CLASS.makeRecord(
                    self, name, level, fn, lno,
                    msg.decode('utf-8', 'ignore'),
                    args, exc_info, func, extra
                )
        return LOGGING_LOGGER_CLASS.makeRecord(
            self, name, level, fn, lno, msg, args, exc_info, func, extra
        )
    # pylint: enable-msg=C0103

# Override the python's logging logger class as soon as this module is imported
if logging.getLoggerClass() is not SaltLoggingClass:

    logging.setLoggerClass(SaltLoggingClass)
    logging.addLevelName(TRACE, 'TRACE')
    logging.addLevelName(GARBAGE, 'GARBAGE')

    if len(logging.root.handlers) == 0:
        # No configuration to the logging system has been done so far.
        # Set the root logger at the lowest level possible
        logging.getLogger().setLevel(GARBAGE)

        # Add a Null logging handler until logging is configured(will be
        # removed at a later stage) so we stop getting:
        #   No handlers could be found for logger "foo"
        logging.getLogger().addHandler(LOGGING_NULL_HANDLER)

def getLogger(name):  # pylint: disable-msg=C0103
    '''
    This function is just a helper, an alias to:
        logging.getLogger(name)

    Although you might find it useful, there's no reason why you should not be
    using the aliased method.
    '''
    return logging.getLogger(name)

def setup_console_logger(log_level='error', log_format=None, date_format=None):
    '''
    Setup the console logger
    '''
    if is_console_configured():
        logging.getLogger(__name__).warn('Console logging already configured')
        return

    # Remove the temporary null logging handler
    __remove_null_logging_handler()

    if log_level is None:
        log_level = 'warning'

    level = LOG_LEVELS.get(log_level.lower(), logging.ERROR)

    handler = None
    for handler in logging.root.handlers:
        if handler.stream is sys.stderr:
            # There's already a logging handler outputting to sys.stderr
            break
    else:
        handler = logging.StreamHandler(sys.stderr)
    handler.setLevel(level)

    # Set the default console formatter config
    if not log_format:
        log_format = '[%(levelname)-8s] %(message)s'
    if not date_format:
        date_format = '%H:%M:%S'

    formatter = logging.Formatter(log_format, datefmt=date_format)

    handler.setFormatter(formatter)
    logging.getLogger().addHandler(handler)

    global __CONSOLE_CONFIGURED
    __CONSOLE_CONFIGURED = True

def setup_logfile_logger(log_path, log_level='error', log_format=None,
                         date_format=None):
    '''
    Setup the logfile logger

    Since version 0.10.6 we support logging to syslog, some examples:

        tcp://localhost:514/LOG_USER
        tcp://localhost/LOG_DAEMON
        udp://localhost:5145/LOG_KERN
        udp://localhost
        file:///dev/log
        file:///dev/log/LOG_SYSLOG
        file:///dev/log/LOG_DAEMON

    The above examples are self explanatory, but:
        <file|udp|tcp>://<host|socketpath>:<port-if-required>/<log-facility>

    If you're thinking on doing remote logging you might also be thinking that
    you could point salt's logging to the remote syslog. **Please Don't!**
    An issue has been reported when doing this over TCP when the logged lines
    get concatenated. See #3061.

    The preferred way to do remote logging is setup a local syslog, point
    salt's logging to the local syslog(unix socket is much faster) and then
    have the local syslog forward the log messages to the remote syslog.
    '''

    if is_logfile_configured():
        logging.getLogger(__name__).warn('Logfile logging already configured')
        return

    if log_path is None:
        logging.getLogger(__name__).warn(
            'log_path setting is set to `None`. Nothing else to do'
        )
        return

    # Remove the temporary null logging handler
    __remove_null_logging_handler()

    if log_level is None:
        log_level = 'warning'

    level = LOG_LEVELS.get(log_level.lower(), logging.ERROR)

    parsed_log_path = urlparse.urlparse(log_path)

    root_logger = logging.getLogger()

    if parsed_log_path.scheme in ('tcp', 'udp', 'file'):
        syslog_opts = {
            'facility': logging.handlers.SysLogHandler.LOG_USER,
            'socktype': socket.SOCK_DGRAM

        }

        if parsed_log_path.scheme == 'file' and parsed_log_path.path:
            facility_name = parsed_log_path.path.split(os.sep)[-1].upper()
            if not facility_name.startswith('LOG_'):
                # The user is not specifying a syslog facility
                facility_name = 'LOG_USER'      # Syslog default
                syslog_opts['address'] = parsed_log_path.path
            else:
                # The user has set a syslog facility, let's update the path to
                # the logging socket
                syslog_opts['address'] = os.sep.join(
                    parsed_log_path.path.split(os.sep)[:-1]
                )
        elif parsed_log_path.path:
            # In case of udp or tcp with a facility specified
            facility_name = parsed_log_path.path.lstrip(os.sep).upper()
            if not facility_name.startswith('LOG_'):
                # Logging facilities start with LOG_ if this is not the case
                # fail right now!
                raise RuntimeError(
                    'The syslog facility {0!r} is not know'.format(
                        facility_name
                    )
                )
        else:
            # This is the case of udp or tcp without a facility specified
            facility_name = 'LOG_USER'      # Syslog default

        facility = getattr(
            logging.handlers.SysLogHandler, facility_name, None
        )
        if facility is None:
            # This python syslog version does not know about the user provided
            # facility name
            raise RuntimeError(
                'The syslog facility {0!r} is not know'.format(
                    facility_name
                )
            )
        syslog_opts['facility'] = facility

        if parsed_log_path.scheme == 'tcp':
            # tcp syslog support was only added on python versions >= 2.7
            if sys.version_info < (2, 7):
                raise RuntimeError(
                    'Python versions lower than 2.7 do not support logging '
                    'to syslog using tcp sockets'
                )
            syslog_opts['socktype'] = socket.SOCK_STREAM

        if parsed_log_path.scheme in ('tcp', 'udp'):
            syslog_opts['address'] = (
                parsed_log_path.hostname,
                parsed_log_path.port or logging.handlers.SYSLOG_UDP_PORT
            )

        if sys.version_info < (2, 7) or parsed_log_path.scheme == 'file':
            # There's not socktype support on python versions lower than 2.7
            syslog_opts.pop('socktype', None)

        # Et voilá! Finally our syslog handler instance
        handler = logging.handlers.SysLogHandler(**syslog_opts)
    else:
        try:
            # Logfile logging is UTF-8 on purpose.
            # Since salt uses yaml and yaml uses either UTF-8 or UTF-16, if a
            # user is not using plain ascii, he's system should be ready to
            # handle UTF-8.
            handler = getattr(
                logging.handlers, 'WatchedFileHandler', logging.FileHandler
            )(log_path, mode='a', encoding='utf-8', delay=0)
        except (IOError, OSError):
            sys.stderr.write(
                'Failed to open log file, do you have permission to write to '
                '{0}\n'.format(log_path)
            )
            sys.exit(2)

    handler.setLevel(level)

    # Set the default console formatter config
    if not log_format:
        log_format = '%(asctime)s [%(name)-15s][%(levelname)-8s] %(message)s'
    if not date_format:
        date_format = '%Y-%m-%d %H:%M:%S'

    formatter = logging.Formatter(log_format, datefmt=date_format)

    handler.setFormatter(formatter)
    root_logger.addHandler(handler)

    global __LOGFILE_CONFIGURED
    __LOGFILE_CONFIGURED = True

def set_logger_level(logger_name, log_level='error'):
    '''
    Tweak a specific logger's logging level
    '''
    logging.getLogger(logger_name).setLevel(
        LOG_LEVELS.get(log_level.lower(), logging.ERROR)
    )

def __remove_null_logging_handler():
    '''
    This function will run once logging has been configured. It just removes
    the NullHandler from the logging handlers.
    '''
    if is_logfile_configured():
        # In this case, the NullHandler has been removed, return!
        return

    root_logger = logging.getLogger()
    global LOGGING_NULL_HANDLER

    for handler in root_logger.handlers:
        if handler is LOGGING_NULL_HANDLER:
            root_logger.removeHandler(LOGGING_NULL_HANDLER)
            # Redefine the null handler to None so it can be garbage collected
            LOGGING_NULL_HANDLER = None
            break

    if sys.version_info >= (2, 7):
        # Python versions >= 2.7 allow warnings to be redirected to the logging
        # system now that it's configured. Let's enable it.
        logging.captureWarnings(True)
<commit_msg>Catch the exception for when salt is unable to connect to syslog.<commit_after># -*- coding: utf-8 -*-
'''
    salt.log
    ~~~~~~~~

    This is where Salt's logging gets set up.

    :copyright: 2011-2012 :email:`Pedro Algarvio (pedro@algarvio.me)`
    :license: Apache 2.0, see LICENSE for more details.
'''

# Import python libs
import os
import re
import sys
import socket
import urlparse
import logging
import logging.handlers

TRACE = logging.TRACE = 5
GARBAGE = logging.GARBAGE = 1

LOG_LEVELS = {
    'all': logging.NOTSET,
    'debug': logging.DEBUG,
    'error': logging.ERROR,
    'garbage': GARBAGE,
    'info': logging.INFO,
    'quiet': 1000,
    'trace': TRACE,
    'warning': logging.WARNING,
}

# Make a list of log level names sorted by log level
SORTED_LEVEL_NAMES = [
    l[0] for l in sorted(LOG_LEVELS.iteritems(), key=lambda x: x[1])
]

# Store an instance of the current logging logger class
LOGGING_LOGGER_CLASS = logging.getLoggerClass()

MODNAME_PATTERN = re.compile(r'(?P<name>%%\(name\)(?:\-(?P<digits>[\d]+))?s)')

__CONSOLE_CONFIGURED = False
__LOGFILE_CONFIGURED = False

def is_console_configured():
    return __CONSOLE_CONFIGURED

def is_logfile_configured():
    return __LOGFILE_CONFIGURED

def is_logging_configured():
    return __CONSOLE_CONFIGURED or __LOGFILE_CONFIGURED

if sys.version_info < (2, 7):
    # Since the NullHandler is only available on python >= 2.7, here's a copy
    class NullHandler(logging.Handler):
        """ This is 1 to 1 copy of python's 2.7 NullHandler"""
        def handle(self, record):
            pass

        def emit(self, record):
            pass

        def createLock(self):  # pylint: disable-msg=C0103
            self.lock = None

    logging.NullHandler = NullHandler

# Store a reference to the null logging handler
LOGGING_NULL_HANDLER = logging.NullHandler()

class LoggingTraceMixIn(object):
    '''
    Simple mix-in class to add a trace method to python's logging.
    '''

    def trace(self, msg, *args, **kwargs):
        self.log(TRACE, msg, *args, **kwargs)

class LoggingGarbageMixIn(object):
    '''
    Simple mix-in class to add a garbage method to python's logging.
    '''

    def garbage(self, msg, *args, **kwargs):
        self.log(GARBAGE, msg, *args, **kwargs)

class LoggingMixInMeta(type):
    '''
    This class is called whenever a new instance of ``SaltLoggingClass`` is
    created.

    What this class does is check if any of the bases have a `trace()` or a
    `garbage()` method defined, if they don't we add the respective mix-ins to
    the bases.
    '''
    def __new__(mcs, name, bases, attrs):
        include_trace = include_garbage = True
        bases = list(bases)
        if name == 'SaltLoggingClass':
            for base in bases:
                if hasattr(base, 'trace'):
                    include_trace = False
                if hasattr(base, 'garbage'):
                    include_garbage = False
        if include_trace:
            bases.append(LoggingTraceMixIn)
        if include_garbage:
            bases.append(LoggingGarbageMixIn)
        return super(LoggingMixInMeta, mcs).__new__(
            mcs, name, tuple(bases), attrs
        )

class SaltLoggingClass(LOGGING_LOGGER_CLASS):
    __metaclass__ = LoggingMixInMeta

    def __new__(mcs, logger_name):
        '''
        We override `__new__` in our logging logger class in order to provide
        some additional features like expand the module name padding if length
        is being used, and also some Unicode fixes.

        This code overhead will only be executed when the class is
        instantiated, ie:

            logging.getLogger(__name__)

        '''
        instance = super(SaltLoggingClass, mcs).__new__(mcs)

        try:
            max_logger_length = len(max(
                logging.Logger.manager.loggerDict.keys(), key=len
            ))
            for handler in logging.getLogger().handlers:
                if handler is LOGGING_NULL_HANDLER:
                    continue

                if not handler.lock:
                    handler.createLock()
                handler.acquire()

                formatter = handler.formatter
                fmt = formatter._fmt.replace('%', '%%')

                match = MODNAME_PATTERN.search(fmt)
                if not match:
                    # Not matched. Release handler and return.
                    handler.release()
                    return instance

                if 'digits' not in match.groupdict():
                    # No digits group. Release handler and return.
                    handler.release()
                    return instance

                digits = match.group('digits')
                if not digits or not (digits and digits.isdigit()):
                    # No valid digits. Release handler and return.
                    handler.release()
                    return instance

                if int(digits) < max_logger_length:
                    # Formatter digits value is lower than current max, update.
                    fmt = fmt.replace(match.group('name'), '%%(name)-%ds')
                    formatter = logging.Formatter(
                        fmt % max_logger_length,
                        datefmt=formatter.datefmt
                    )
                    handler.setFormatter(formatter)
                handler.release()
        except ValueError:
            # There are no registered loggers yet
            pass
        return instance

    # pylint: disable-msg=C0103
    def makeRecord(self, name, level, fn, lno, msg, args, exc_info, func=None,
                   extra=None):
        # Let's try to make every logging message unicode
        if isinstance(msg, basestring) and not isinstance(msg, unicode):
            try:
                return LOGGING_LOGGER_CLASS.makeRecord(
                    self, name, level, fn, lno,
                    msg.decode('utf-8', 'replace'),
                    args, exc_info, func, extra
                )
            except UnicodeDecodeError:
                return LOGGING_LOGGER_CLASS.makeRecord(
                    self, name, level, fn, lno,
                    msg.decode('utf-8', 'ignore'),
                    args, exc_info, func, extra
                )
        return LOGGING_LOGGER_CLASS.makeRecord(
            self, name, level, fn, lno, msg, args, exc_info, func, extra
        )
    # pylint: enable-msg=C0103

# Override the python's logging logger class as soon as this module is imported
if logging.getLoggerClass() is not SaltLoggingClass:

    logging.setLoggerClass(SaltLoggingClass)
    logging.addLevelName(TRACE, 'TRACE')
    logging.addLevelName(GARBAGE, 'GARBAGE')

    if len(logging.root.handlers) == 0:
        # No configuration to the logging system has been done so far.
        # Set the root logger at the lowest level possible
        logging.getLogger().setLevel(GARBAGE)

        # Add a Null logging handler until logging is configured(will be
        # removed at a later stage) so we stop getting:
        #   No handlers could be found for logger "foo"
        logging.getLogger().addHandler(LOGGING_NULL_HANDLER)

def getLogger(name):  # pylint: disable-msg=C0103
    '''
    This function is just a helper, an alias to:
        logging.getLogger(name)

    Although you might find it useful, there's no reason why you should not be
    using the aliased method.
    '''
    return logging.getLogger(name)

def setup_console_logger(log_level='error', log_format=None, date_format=None):
    '''
    Setup the console logger
    '''
    if is_console_configured():
        logging.getLogger(__name__).warn('Console logging already configured')
        return

    # Remove the temporary null logging handler
    __remove_null_logging_handler()

    if log_level is None:
        log_level = 'warning'

    level = LOG_LEVELS.get(log_level.lower(), logging.ERROR)

    handler = None
    for handler in logging.root.handlers:
        if handler.stream is sys.stderr:
            # There's already a logging handler outputting to sys.stderr
            break
    else:
        handler = logging.StreamHandler(sys.stderr)
    handler.setLevel(level)

    # Set the default console formatter config
    if not log_format:
        log_format = '[%(levelname)-8s] %(message)s'
    if not date_format:
        date_format = '%H:%M:%S'

    formatter = logging.Formatter(log_format, datefmt=date_format)

    handler.setFormatter(formatter)
    logging.getLogger().addHandler(handler)

    global __CONSOLE_CONFIGURED
    __CONSOLE_CONFIGURED = True

def setup_logfile_logger(log_path, log_level='error', log_format=None,
                         date_format=None):
    '''
    Setup the logfile logger

    Since version 0.10.6 we support logging to syslog, some examples:

        tcp://localhost:514/LOG_USER
        tcp://localhost/LOG_DAEMON
        udp://localhost:5145/LOG_KERN
        udp://localhost
        file:///dev/log
        file:///dev/log/LOG_SYSLOG
        file:///dev/log/LOG_DAEMON

    The above examples are self explanatory, but:
        <file|udp|tcp>://<host|socketpath>:<port-if-required>/<log-facility>

    If you're thinking on doing remote logging you might also be thinking that
    you could point salt's logging to the remote syslog. **Please Don't!**
    An issue has been reported when doing this over TCP when the logged lines
    get concatenated. See #3061.

    The preferred way to do remote logging is setup a local syslog, point
    salt's logging to the local syslog(unix socket is much faster) and then
    have the local syslog forward the log messages to the remote syslog.
    '''

    if is_logfile_configured():
        logging.getLogger(__name__).warn('Logfile logging already configured')
        return

    if log_path is None:
        logging.getLogger(__name__).warn(
            'log_path setting is set to `None`. Nothing else to do'
        )
        return

    # Remove the temporary null logging handler
    __remove_null_logging_handler()

    if log_level is None:
        log_level = 'warning'

    level = LOG_LEVELS.get(log_level.lower(), logging.ERROR)

    parsed_log_path = urlparse.urlparse(log_path)

    root_logger = logging.getLogger()

    if parsed_log_path.scheme in ('tcp', 'udp', 'file'):
        syslog_opts = {
            'facility': logging.handlers.SysLogHandler.LOG_USER,
            'socktype': socket.SOCK_DGRAM

        }

        if parsed_log_path.scheme == 'file' and parsed_log_path.path:
            facility_name = parsed_log_path.path.split(os.sep)[-1].upper()
            if not facility_name.startswith('LOG_'):
                # The user is not specifying a syslog facility
                facility_name = 'LOG_USER'      # Syslog default
                syslog_opts['address'] = parsed_log_path.path
            else:
                # The user has set a syslog facility, let's update the path to
                # the logging socket
                syslog_opts['address'] = os.sep.join(
                    parsed_log_path.path.split(os.sep)[:-1]
                )
        elif parsed_log_path.path:
            # In case of udp or tcp with a facility specified
            facility_name = parsed_log_path.path.lstrip(os.sep).upper()
            if not facility_name.startswith('LOG_'):
                # Logging facilities start with LOG_ if this is not the case
                # fail right now!
                raise RuntimeError(
                    'The syslog facility {0!r} is not know'.format(
                        facility_name
                    )
                )
        else:
            # This is the case of udp or tcp without a facility specified
            facility_name = 'LOG_USER'      # Syslog default

        facility = getattr(
            logging.handlers.SysLogHandler, facility_name, None
        )
        if facility is None:
            # This python syslog version does not know about the user provided
            # facility name
            raise RuntimeError(
                'The syslog facility {0!r} is not know'.format(
                    facility_name
                )
            )
        syslog_opts['facility'] = facility

        if parsed_log_path.scheme == 'tcp':
            # tcp syslog support was only added on python versions >= 2.7
            if sys.version_info < (2, 7):
                raise RuntimeError(
                    'Python versions lower than 2.7 do not support logging '
                    'to syslog using tcp sockets'
                )
            syslog_opts['socktype'] = socket.SOCK_STREAM

        if parsed_log_path.scheme in ('tcp', 'udp'):
            syslog_opts['address'] = (
                parsed_log_path.hostname,
                parsed_log_path.port or logging.handlers.SYSLOG_UDP_PORT
            )

        if sys.version_info < (2, 7) or parsed_log_path.scheme == 'file':
            # There's not socktype support on python versions lower than 2.7
            syslog_opts.pop('socktype', None)

        try:
            # Et voilá! Finally our syslog handler instance
            handler = logging.handlers.SysLogHandler(**syslog_opts)
        except socket.error, err:
            logging.getLogger(__name__).error(
                'Failed to setup the Syslog logging handler: {0}'.format(
                    err
                )
            )
            # Do not proceed with any more configuration since it will fail, we
            # have the console logging already setup and the user should see
            # the error.
            return
    else:
        try:
            # Logfile logging is UTF-8 on purpose.
            # Since salt uses yaml and yaml uses either UTF-8 or UTF-16, if a
            # user is not using plain ascii, he's system should be ready to
            # handle UTF-8.
            handler = getattr(
                logging.handlers, 'WatchedFileHandler', logging.FileHandler
            )(log_path, mode='a', encoding='utf-8', delay=0)
        except (IOError, OSError):
            sys.stderr.write(
                'Failed to open log file, do you have permission to write to '
                '{0}\n'.format(log_path)
            )
            sys.exit(2)

    handler.setLevel(level)

    # Set the default console formatter config
    if not log_format:
        log_format = '%(asctime)s [%(name)-15s][%(levelname)-8s] %(message)s'
    if not date_format:
        date_format = '%Y-%m-%d %H:%M:%S'

    formatter = logging.Formatter(log_format, datefmt=date_format)

    handler.setFormatter(formatter)
    root_logger.addHandler(handler)

    global __LOGFILE_CONFIGURED
    __LOGFILE_CONFIGURED = True

def set_logger_level(logger_name, log_level='error'):
    '''
    Tweak a specific logger's logging level
    '''
    logging.getLogger(logger_name).setLevel(
        LOG_LEVELS.get(log_level.lower(), logging.ERROR)
    )

def __remove_null_logging_handler():
    '''
    This function will run once logging has been configured. It just removes
    the NullHandler from the logging handlers.
    '''
    if is_logfile_configured():
        # In this case, the NullHandler has been removed, return!
        return

    root_logger = logging.getLogger()
    global LOGGING_NULL_HANDLER

    for handler in root_logger.handlers:
        if handler is LOGGING_NULL_HANDLER:
            root_logger.removeHandler(LOGGING_NULL_HANDLER)
            # Redefine the null handler to None so it can be garbage collected
            LOGGING_NULL_HANDLER = None
            break

    if sys.version_info >= (2, 7):
        # Python versions >= 2.7 allow warnings to be redirected to the logging
        # system now that it's configured. Let's enable it.
        logging.captureWarnings(True)
<|endoftext|>


<commit_before>#!/usr/bin/env python
import os
import shutil
import sys
import subprocess
import json
import yaml
import inflect
from scaffold.custom_fields import *
from scaffold.replace_string import *
from scaffold.modules.errors import BlueprintError
from scaffold.angular_io import create_angular_files

blueprint_file = 'app/__init__.py'
test_script = 'tests.bash'
yaml_file = sys.argv[1]
# Angular 6 files
app_js_file = "app/templates/static/src/app/app.module.ts"

#change var name to layout file
sidebar_file = "app/templates/static/src/app/layout/components/sidebar/sidebar.component.html"
layout_route_file = "app/templates/static/src/app/layout/layout-routing.module.ts"
#end js layout

#Path to files for scaffolding js files
path_scaffold_template = "scaffold/app/templates/"
conf_js_file = "conf.js"

# Error classes
def make_plural(resource):
    # https://pypi.python.org/pypi/inflect
    p = inflect.engine()
    if p.singular_noun(resource):
        resources = resource
        resource = p.singular_noun(resource)
        return resource, resources
    else:
        resources = p.plural(resource)
        return resource, resources

#Generate Flask API files
def generate_files(module_path):

    app_files = ['views.py', 'models.py', '__init__.py',  'tests.py']

    for file in app_files:

        # Generate App files
        if file == "views.py":
            with open(os.path.join(module_path, 'views.py'), "w") as new_file:
                with open("scaffold/app/views.py", "r") as old_file:
                    for line in old_file:
                        new_file.write(line.format(resource=resource,
                                                   resources=resources,
                                                   Resources=resources.title(),
                                                   Resource=resource.title(),
                                                   add_fields=add_fields))

        elif file == "models.py":
            with open(os.path.join(module_path, 'models.py'), "w") as new_file:
                with open("scaffold/app/models.py", "r") as old_file:
                    for line in old_file:
                        new_file.write(line.format(resource=resource, resources=resources,
                                                   Resources=resources.title(),
                                                   db_rows=db_rows,
                                                   schema=schema, meta=meta,
                                                   init_self_vars=init_self_vars,
                                                   init_args=init_args))

        elif file == "__init__.py":
            with open(os.path.join(module_path, '__init__.py'), "w") as new_file:
                with open("scaffold/app/__init__.py", "r") as old_file:
                    for line in old_file:
                        new_file.write(line)

def register_blueprints():
    string_to_insert_after = '# Blueprints'
    new_blueprint = """
    # Blueprints
    from app.{resources}.views import {resources}
    app.register_blueprint({resources}, url_prefix='/api/v1/{resources}')""".format(resources=resources)

    with open(blueprint_file, 'r+') as old_file:
        filedata = old_file.read()
    if string_to_insert_after in filedata:
        # replace the first occurrence
        new_filedata = filedata.replace(
            string_to_insert_after, new_blueprint, 1)
        with open(blueprint_file, 'w') as new_file:
            new_file.write(new_filedata)
            print("Registered Blueprints for ", resources)
    else:
        raise BlueprintError()

def clean_up(module_path):
    if os.path.isdir(module_path):
        shutil.rmtree(module_path)

#change to exclude template folder
def run_autopep8():
    try:
        cmd_output = subprocess.check_output(
            ['autopep8', '--in-place', '--recursive', 'app'])
        print("Ran autopep8")
    except subprocess.CalledProcessError:
        print("autopep8 failed")
        raise

def run_ngbuild():
    try:
        os.chdir("app/templates/static/")
        cmd_output = subprocess.check_output(
            ['ng', 'build', '--prod'])
        print("Ran Ng Build --prod")
    except subprocess.CalledProcessError:
        print("Ng build --prod failed")
        raise

# Main Code Start
#
# Parse YAML file
with open(yaml_file, "r") as file:

    yaml_data = yaml.load(file)

    for module, fields in yaml_data.items():
            # make module name plural
        resource, resources = make_plural(module)

        # Start strings to insert into models
        db_rows = ""
        schema = ""
        meta = ""
        init_self_vars = ""
        init_args = ""
        # End strings to insert into models

        # Start strings to insert into views
        add_fields = ""

        for f in fields:
            field, field_type = f.split(':')
            if field_type == "string":
                db_rows += """
    {} = db.Column(db.String(250), nullable=False)""".format(field)
                schema += """
    {} = fields.String(validate=not_blank)""".format(field)

            elif field_type == "boolean":
                db_rows += """
    {} = db.Column(db.Integer, nullable=False)""".format(field)
                schema += """
    {} = fields.Integer(required=True)""".format(field)

            elif field_type == "integer":
                db_rows += """
    {} = db.Column(db.Integer, nullable=False)""".format(field)
                schema += """
    {} = fields.Integer(required=True)""".format(field)

            elif field_type == "email":
                db_rows += """
    {} = db.Column(db.String(250), nullable=False)""".format(field)
                schema += """
    {} = fields.Email(validate=not_blank)""".format(field)

            elif field_type == "url":
                db_rows += """
    {} = db.Column(db.String(250), nullable=False)""".format(field)
                schema += """
    {} = fields.URL(validate=not_blank)""".format(field)

            #Will be auto populated at the db level
            elif field_type == "datetime":
                db_rows += """
    {} = db.Column(db.TIMESTAMP,server_default=db.func.current_timestamp(),nullable=False)""".format(field)
                schema += """
    {} = fields.DateTime()""".format(field)
                continue

            elif field_type == "date":
                db_rows += """
    {} = db.Column(db.Date, nullable=False)""".format(field)
                schema += """
    {} = fields.Date(required=True)""".format(field)

            elif field_type == "decimal":
                db_rows += """
    {} = db.Column(db.Numeric, nullable=False)""".format(field)
                schema += """
    {} = fields.Decimal(as_string=True)""".format(field)

            elif field_type == "text":
                db_rows += """
    {} = db.Column(db.Text, nullable=False)""".format(field)
                schema += """
    {} = fields.String(validate=not_blank)""".format(field)

            # models
            meta += """ '{}', """.format(field)
            init_args += """ {}, """.format(field)
            init_self_vars += """
        self.{field} = {field}""".format(field=field)
            # Views
            add_fields += add_string.format(field)

        module_dir = os.path.join('app', resources)
        try:
            os.mkdir(module_dir)
            try:

                generate_files(module_dir)
                create_angular_files(module, fields)
                print("files created successfully")
                register_blueprints()
                # Update service in app.module.ts
                replace_string(resource, resources , app_js_file, "// service", app_module_string)
                replace_string(resource, resources , app_js_file, "// import", app_import_string)

                # Add js files to index.html
                replace_string(
                    resource, resources, layout_route_file, "// route", layout_route_string)

                # Add menus to the sidebar.html
                replace_string(
                    resource, resources, sidebar_file, "<!-- menu -->", menu_string)
                # Add routes __init__.py
                replace_string(resource,resources,blueprint_file, "#app-route", app_route)

                run_ngbuild()
                #run_autopep8()
            except:
                clean_up(module_dir)
                raise

        except:
            raise
<commit_msg>datetime<commit_after>#!/usr/bin/env python
import os
import shutil
import sys
import subprocess
import json
import yaml
import inflect
from scaffold.custom_fields import *
from scaffold.replace_string import *
from scaffold.modules.errors import BlueprintError
from scaffold.angular_io import create_angular_files

blueprint_file = 'app/__init__.py'
test_script = 'tests.bash'
yaml_file = sys.argv[1]
# Angular 6 files
app_js_file = "app/templates/static/src/app/app.module.ts"

#change var name to layout file
sidebar_file = "app/templates/static/src/app/layout/components/sidebar/sidebar.component.html"
layout_route_file = "app/templates/static/src/app/layout/layout-routing.module.ts"
#end js layout

#Path to files for scaffolding js files
path_scaffold_template = "scaffold/app/templates/"
conf_js_file = "conf.js"

# Error classes
def make_plural(resource):
    # https://pypi.python.org/pypi/inflect
    p = inflect.engine()
    if p.singular_noun(resource):
        resources = resource
        resource = p.singular_noun(resource)
        return resource, resources
    else:
        resources = p.plural(resource)
        return resource, resources

#Generate Flask API files
def generate_files(module_path):

    app_files = ['views.py', 'models.py', '__init__.py',  'tests.py']

    for file in app_files:

        # Generate App files
        if file == "views.py":
            with open(os.path.join(module_path, 'views.py'), "w") as new_file:
                with open("scaffold/app/views.py", "r") as old_file:
                    for line in old_file:
                        new_file.write(line.format(resource=resource,
                                                   resources=resources,
                                                   Resources=resources.title(),
                                                   Resource=resource.title(),
                                                   add_fields=add_fields))

        elif file == "models.py":
            with open(os.path.join(module_path, 'models.py'), "w") as new_file:
                with open("scaffold/app/models.py", "r") as old_file:
                    for line in old_file:
                        new_file.write(line.format(resource=resource, resources=resources,
                                                   Resources=resources.title(),
                                                   db_rows=db_rows,
                                                   schema=schema, meta=meta,
                                                   init_self_vars=init_self_vars,
                                                   init_args=init_args))

        elif file == "__init__.py":
            with open(os.path.join(module_path, '__init__.py'), "w") as new_file:
                with open("scaffold/app/__init__.py", "r") as old_file:
                    for line in old_file:
                        new_file.write(line)

def register_blueprints():
    string_to_insert_after = '# Blueprints'
    new_blueprint = """
    # Blueprints
    from app.{resources}.views import {resources}
    app.register_blueprint({resources}, url_prefix='/api/v1/{resources}')""".format(resources=resources)

    with open(blueprint_file, 'r+') as old_file:
        filedata = old_file.read()
    if string_to_insert_after in filedata:
        # replace the first occurrence
        new_filedata = filedata.replace(
            string_to_insert_after, new_blueprint, 1)
        with open(blueprint_file, 'w') as new_file:
            new_file.write(new_filedata)
            print("Registered Blueprints for ", resources)
    else:
        raise BlueprintError()

def clean_up(module_path):
    if os.path.isdir(module_path):
        shutil.rmtree(module_path)

#change to exclude template folder
def run_autopep8():
    try:
        cmd_output = subprocess.check_output(
            ['autopep8', '--in-place', '--recursive', 'app'])
        print("Ran autopep8")
    except subprocess.CalledProcessError:
        print("autopep8 failed")
        raise

def run_ngbuild():
    try:
        os.chdir("app/templates/static/")
        cmd_output = subprocess.check_output(
            ['ng', 'build', '--prod'])
        print("Ran Ng Build --prod")
    except subprocess.CalledProcessError:
        print("Ng build --prod failed")
        raise

# Main Code Start
#
# Parse YAML file
with open(yaml_file, "r") as file:

    yaml_data = yaml.load(file)

    for module, fields in yaml_data.items():
            # make module name plural
        resource, resources = make_plural(module)

        # Start strings to insert into models
        db_rows = ""
        schema = ""
        meta = ""
        init_self_vars = ""
        init_args = ""
        # End strings to insert into models

        # Start strings to insert into views
        add_fields = ""

        for f in fields:
            field, field_type = f.split(':')
            if field_type == "string":
                db_rows += """
    {} = db.Column(db.String(250), nullable=False)""".format(field)
                schema += """
    {} = fields.String(validate=not_blank)""".format(field)

            elif field_type == "boolean":
                db_rows += """
    {} = db.Column(db.Integer, nullable=False)""".format(field)
                schema += """
    {} = fields.Integer(required=True)""".format(field)

            elif field_type == "integer":
                db_rows += """
    {} = db.Column(db.Integer, nullable=False)""".format(field)
                schema += """
    {} = fields.Integer(required=True)""".format(field)

            elif field_type == "email":
                db_rows += """
    {} = db.Column(db.String(250), nullable=False)""".format(field)
                schema += """
    {} = fields.Email(validate=not_blank)""".format(field)

            elif field_type == "url":
                db_rows += """
    {} = db.Column(db.String(250), nullable=False)""".format(field)
                schema += """
    {} = fields.URL(validate=not_blank)""".format(field)

            #Will be auto populated at the db level
            elif field_type == "datetime":
                db_rows += """
    {} = db.Column(db.TIMESTAMP,nullable=False, default=datetime.utcnow)""".format(field)
                schema += """
    {} = fields.DateTime()""".format(field)
                continue

            elif field_type == "date":
                db_rows += """
    {} = db.Column(db.Date, nullable=False)""".format(field)
                schema += """
    {} = fields.Date(required=True)""".format(field)

            elif field_type == "decimal":
                db_rows += """
    {} = db.Column(db.Numeric, nullable=False)""".format(field)
                schema += """
    {} = fields.Decimal(as_string=True)""".format(field)

            elif field_type == "text":
                db_rows += """
    {} = db.Column(db.Text, nullable=False)""".format(field)
                schema += """
    {} = fields.String(validate=not_blank)""".format(field)

            # models
            meta += """ '{}', """.format(field)
            init_args += """ {}, """.format(field)
            init_self_vars += """
        self.{field} = {field}""".format(field=field)
            # Views
            add_fields += add_string.format(field)

        module_dir = os.path.join('app', resources)
        try:
            os.mkdir(module_dir)
            try:

                generate_files(module_dir)
                create_angular_files(module, fields)
                print("files created successfully")
                register_blueprints()
                # Update service in app.module.ts
                replace_string(resource, resources , app_js_file, "// service", app_module_string)
                replace_string(resource, resources , app_js_file, "// import", app_import_string)

                # Add js files to index.html
                replace_string(
                    resource, resources, layout_route_file, "// route", layout_route_string)

                # Add menus to the sidebar.html
                replace_string(
                    resource, resources, sidebar_file, "<!-- menu -->", menu_string)
                # Add routes __init__.py
                replace_string(resource,resources,blueprint_file, "#app-route", app_route)

                run_ngbuild()
                #run_autopep8()
            except:
                clean_up(module_dir)
                raise

        except:
            raise
<|endoftext|>


<commit_before>#! /usr/bin/python

# Scans the LAN for all hosts and reports some statistics about them

import MySQLdb as mdb
import subprocess as sp
import sys
import syslog
import time
import traceback

# compare the gathered data to what is in the database
def lstvssql(hostlist):
  try:
    lastseen = time.strftime('%Y-%m-%d %H:%M:%S')
    # connect to the database
    con = mdb.connect(host='sql.lan', db='dhcpnsdb', read_default_file='~/.dns.cnf')
    # activate a cursor
    cur = con.cursor()
    # test the connection
    cur.execute("SELECT VERSION()")
    ver = cur.fetchone()
    print "SQL database version: ", ver
    print "Scan performed on:    ", lastseen
    for idx, line in enumerate(hostlist):
      mac = line[3]
      ipoctet4 = str(line[8]).zfill(3)
      nodename = line[1]
      # check if MAC exists in DB
      if (len(mac) == 17):
        # MAC is valid: search for it in the DB
        cmd = ('SELECT * '
               'FROM lantbl '
               'WHERE mac="' + mac + '"')
        cur.execute(cmd)
        rsl = cur.fetchone()
        if (rsl is None):
          if DEBUG:
            print "MAC is not found in DB"
          if (line[5] != 0):
            if DEBUG:
              print "... new host found"
            # & host is pingable -> new host, so add it to the DB
            cmd = ('INSERT INTO lantbl '
                   '(mac, ipoctet4, lastseen, nodename) '
                   'VALUES (%s, %s, %s, %s)')
            dat = (mac, ipoctet4, lastseen, nodename)
            syslog.syslog(syslog.LOG_NOTICE, "INSERTed " + mac + " @ " + nodename)
            cur.execute(cmd, dat)
            con.commit()
            line[10] = lastseen
          # {endif}
        else:
          if DEBUG:
            print "MAC is present in DB"
          if (line[5] != 0):
            if DEBUG:
              print "... updating existing hostdata"
            cmd = ('UPDATE lantbl '
                   'SET lastseen = %s, nodename = %s, ipoctet4 = %s '
                   'WHERE mac = %s ')
            dat = (lastseen, nodename, ipoctet4, mac)
            cur.execute(cmd, dat)
            con.commit()
            line[10] = lastseen
          else:
            # & host is not pingable -> update local data (arp-data may be stale)
            # print "exists in DB; not pingable. Local data may not be up-to-date."
            # print "update info ", mac, rsl
            # print line[1],line[2]
            line[1] = "* " + rsl[3]
            line[2] = "* " + rsl[3]
            line[10] = rsl[2]
          # {endif}
        # {endif}
      else:
        # sometimes nodename = "?" and mac = "<incomplete>"
        if (nodename == "?"):
          # then lookup the last user of the IP-address
          cmd = ('SELECT * '
                 'FROM lantbl '
                 'WHERE ipoctet4="' + ipoctet4 + '"')
          cur.execute(cmd)
          rsl = cur.fetchone()
          # example output
          # rsl <= ('00:00:00:00:00:00', '182', datetime.datetime(2015, 10, 18, 14, 45, 26), 'hostname')
          if (rsl is not None):
            line[1] = "-" + rsl[3]
            line[2] = "-" + rsl[3]
            line[3] = "-" + rsl[0]
            line[10] = rsl[2]
        else:
          print nodename, mac, ipoctet4
      # {endif}
    # {endfor}
  except mdb.Error, e:
    print e.__doc__
    syslog.syslog(syslog.LOG_ALERT, e.__doc__)
    syslog_trace(traceback.format_exc())
  finally:
    if con:
        cur.close()
        con.close()
  # {endtry}
  return hostlist

# read the system UN*X epoch
def getuxtime():
  cmd = ["date", "+'%s'"]
  dt = sp.Popen(cmd, stdout=sp.PIPE, stderr=sp.PIPE)
  output, err = dt.communicate()
  entries = output.replace("'", "").splitlines()
  return entries

# read the contents /var/lib/misc/dnsmasq.leases
def getleases(listsize):
  hostlist = []
  fi = "/var/lib/misc/dnsmasq.leases"
  f    = file(fi, 'r')
  cat = f.read().strip('\n')
  f.close()
  entries = cat.splitlines()
  if DEBUG:
    print entries

  # fill the array with datafrom the leases
  for idx, line in enumerate(entries):
    if DEBUG:
      print idx, line
    hostlist.extend([[None] * listsize])
    items = line.split()
    # IP
    ip = items[2]
    hostlist[idx][0] = ip
    # hostlist[idx][8] = int(ip.split('.')[3])
    # hostname
    hostlist[idx][1] = items[3]
    # MAC
    hostlist[idx][3] = items[1]
    # T2R
    hostlist[idx][9] = (int(items[0]) - ux)/60

  return hostlist

# get the contents of the arp table (arp -a)
def getarp(hostlist):
  listsize = len(hostlist[0])
  cmd = ["/usr/sbin/arp", "-a"]
  arp = sp.Popen(cmd, stdout=sp.PIPE, stderr=sp.PIPE)
  output, err = arp.communicate()
  if DEBUG:
    print err
  if DEBUG:
    print output
  entries = output.splitlines()

  # make a list of the IPs
  column0list = [hostlist[i][0] for i in xrange(len(hostlist))]
  if DEBUG:
    print "\t", column0list

  # Add `arp` data to the array
  for idx, line in enumerate(entries):
    if DEBUG:
      print idx, line
    items = line.split()
    # IP according to arp
    ip = items[1][1:-1]
    try:
      adx = column0list.index(ip)
      # arp hostname
      hostlist[adx][2] = items[0]
      hostlist[adx][8] = int(ip.split('.')[3])
    except ValueError:
      hostlist.extend([[None] * listsize])
      adx = len(hostlist)-1
      hostlist[adx][0] = ip
      hostlist[adx][8] = int(ip.split('.')[3])
      hostlist[adx][1] = items[0]
      hostlist[adx][3] = items[3]
      hostlist[adx][2] = items[0]
      hostlist[adx][9] = -1
      column0list.extend([hostlist[i][0]])

  return hostlist

# ping each host in the list and store the timings
def pingpong(hostlist):
  for idx, line in enumerate(hostlist):
    ip = line[0]
    pong = map(float, ping(ip, 1))
    if pong[0] > 0:
      pong = map(float, ping(ip, 10))
    hostlist[idx][4] = pong[0]
    hostlist[idx][5] = pong[1]
    hostlist[idx][6] = pong[2]
    hostlist[idx][7] = pong[3]
    if (pong[0] == 0) and (hostlist[idx][9] is None):
        hostlist[idx][9] = 0

  return hostlist

# ping the given `ip`  `cnt` times and return responsetimes
def ping(ip, cnt):
  cmd = ["ping", "-w", "1", "-q", "-i", "0.5", "-c", str(cnt), ip]
  ping = sp.Popen(cmd, stdout=sp.PIPE, stderr=sp.PIPE)
  output, err = ping.communicate()

  # get last line of output
  line = output.splitlines()[-1]
  # => rtt min/avg/max/mdev = 1.069/1.257/1.777/0.302 ms
  if (len(line) < 12):
    line = 'rtt min/avg/max/mdev = 0.00/0.00/0.00/0.00 ms'

  # get third field
  field3 = line.split()[3]
  # ==> 1.069/1.257/1.777/0.302
  # split the field at "/"
  result = field3.split('/')
  # ===> ['1.036', '1.224', '1.496', '0.171']
  return result

# 'row' in list to sort by
def getkey(item):
  return item[8]

# format a string to make it show red on an ANSI terminal
def red(text):
  print ("\033[91m {}\033[00m" .format(text))

# Log a python stack trace to syslog
def syslog_trace(trace):
  log_lines = trace.split('\n')
  for line in log_lines:
    if line:
      syslog.syslog(syslog.LOG_ALERT, line)

if __name__ == '__main__':
  DEBUG = False
  lsa = len(sys.argv)
  if (lsa == 1):
    sw = 2
  else:
    sw = 2
    if (sys.argv[1] == '-t'):
      sw = 1
    # {endif}
  # {endif}
  try:
    ux = getuxtime()
    ux = map(int, ux)[0]

    #  0 = IP
    #  1 = hostname (dhcp lease)
    #  2 = hostname (arp)
    #  3 = MAC
    #  4 = ping min
    #  5 = ping avg
    #  6 = ping max
    #  7 = ping stdev
    #  8 = IP(...4)
    #  9 = Time to release (minutes)
    # 10 = lastseen
    hostlist = getleases(11)  # parameter is size of the array
    if DEBUG:
      print len(hostlist), "\n"

    hostlist = getarp(hostlist)  # add the hosts that no longer have a lease but are still present in the arp cache
    if DEBUG:
      print len(hostlist), "\n"

    hostlist = sorted(hostlist, key=getkey)  # sort the list by IP octet 4

    hostlist = pingpong(hostlist)  # search for signs of life

    hostlist = lstvssql(hostlist)  # compare the list with the database

    # determine fieldlength of hostname for printing.
    lenhost = 0
    for idx, line in enumerate(hostlist):
      if len(hostlist[idx][1]) > lenhost:
        lenhost = len(hostlist[idx][1])
    # {endfor}

    for idx, line in enumerate(hostlist):
      spc0 = ' ' * (16 - len(line[0]))
      spc1 = ' ' * (lenhost - len(line[1]) + 1)
      spc2 = ' ' * (17 - len(line[3]) + 1)
      if (sw == 1):
        print line[0], spc0, line[1], spc1, line[3], spc2, "avg=", line[5], "\tstdev=", line[7], "\tT2R=", line[9]
      if (sw == 2):
        print line[0], spc0, line[1], spc1, line[3], spc2, "last seen :", line[10]
    # {endfor}

  except Exception as e:
    if DEBUG:
      print("Unexpected error:")
      red(e.message)
    syslog.syslog(syslog.LOG_ALERT, e.__doc__)
    syslog_trace(traceback.format_exc())
    raise
  # {endtry}
<commit_msg>Put function descriptions in triple quotes and inside the function<commit_after>#! /usr/bin/python

"""Scans the LAN for all hosts and reports some statistics about them"""

import MySQLdb as mdb
import subprocess as sp
import sys
import syslog
import time
import traceback

def lstvssql(hostlist):
  """Compare the gathered data to what is in the database."""
  try:
    lastseen = time.strftime('%Y-%m-%d %H:%M:%S')
    # connect to the database
    con = mdb.connect(host='sql.lan', db='dhcpnsdb', read_default_file='~/.dns.cnf')
    # activate a cursor
    cur = con.cursor()
    # test the connection
    cur.execute("SELECT VERSION()")
    ver = cur.fetchone()
    print "SQL database version: ", ver
    print "Scan performed on:    ", lastseen
    for idx, line in enumerate(hostlist):
      mac = line[3]
      ipoctet4 = str(line[8]).zfill(3)
      nodename = line[1]
      # check if MAC exists in DB
      if (len(mac) == 17):
        # MAC is valid: search for it in the DB
        cmd = ('SELECT * '
               'FROM lantbl '
               'WHERE mac="' + mac + '"')
        cur.execute(cmd)
        rsl = cur.fetchone()
        if (rsl is None):
          if DEBUG:
            print "MAC is not found in DB"
          if (line[5] != 0):
            if DEBUG:
              print "... new host found"
            # & host is pingable -> new host, so add it to the DB
            cmd = ('INSERT INTO lantbl '
                   '(mac, ipoctet4, lastseen, nodename) '
                   'VALUES (%s, %s, %s, %s)')
            dat = (mac, ipoctet4, lastseen, nodename)
            syslog.syslog(syslog.LOG_NOTICE, "INSERTed " + mac + " @ " + nodename)
            cur.execute(cmd, dat)
            con.commit()
            line[10] = lastseen
          # {endif}
        else:
          if DEBUG:
            print "MAC is present in DB"
          if (line[5] != 0):
            if DEBUG:
              print "... updating existing hostdata"
            cmd = ('UPDATE lantbl '
                   'SET lastseen = %s, nodename = %s, ipoctet4 = %s '
                   'WHERE mac = %s ')
            dat = (lastseen, nodename, ipoctet4, mac)
            cur.execute(cmd, dat)
            con.commit()
            line[10] = lastseen
          else:
            # & host is not pingable -> update local data (arp-data may be stale)
            # print "exists in DB; not pingable. Local data may not be up-to-date."
            # print "update info ", mac, rsl
            # print line[1],line[2]
            line[1] = "* " + rsl[3]
            line[2] = "* " + rsl[3]
            line[10] = rsl[2]
          # {endif}
        # {endif}
      else:
        # sometimes nodename = "?" and mac = "<incomplete>"
        if (nodename == "?"):
          # then lookup the last user of the IP-address
          cmd = ('SELECT * '
                 'FROM lantbl '
                 'WHERE ipoctet4="' + ipoctet4 + '"')
          cur.execute(cmd)
          rsl = cur.fetchone()
          # example output
          # rsl <= ('00:00:00:00:00:00', '182', datetime.datetime(2015, 10, 18, 14, 45, 26), 'hostname')
          if (rsl is not None):
            line[1] = "-" + rsl[3]
            line[2] = "-" + rsl[3]
            line[3] = "-" + rsl[0]
            line[10] = rsl[2]
        else:
          print nodename, mac, ipoctet4
      # {endif}
    # {endfor}
  except mdb.Error, e:
    print e.__doc__
    syslog.syslog(syslog.LOG_ALERT, e.__doc__)
    syslog_trace(traceback.format_exc())
  finally:
    if con:
        cur.close()
        con.close()
  # {endtry}
  return hostlist

def getuxtime():
  """Read the system UN*X epoch"""
  cmd = ["date", "+'%s'"]
  dt = sp.Popen(cmd, stdout=sp.PIPE, stderr=sp.PIPE)
  output, err = dt.communicate()
  entries = output.replace("'", "").splitlines()
  return entries

def getleases(listsize):
  """Read the contents /var/lib/misc/dnsmasq.leases"""
  hostlist = []
  fi = "/var/lib/misc/dnsmasq.leases"
  f    = file(fi, 'r')
  cat = f.read().strip('\n')
  f.close()
  entries = cat.splitlines()
  if DEBUG:
    print entries

  # fill the array with datafrom the leases
  for idx, line in enumerate(entries):
    if DEBUG:
      print idx, line
    hostlist.extend([[None] * listsize])
    items = line.split()
    # IP
    ip = items[2]
    hostlist[idx][0] = ip
    # hostlist[idx][8] = int(ip.split('.')[3])
    # hostname
    hostlist[idx][1] = items[3]
    # MAC
    hostlist[idx][3] = items[1]
    # T2R
    hostlist[idx][9] = (int(items[0]) - ux)/60

  return hostlist

def getarp(hostlist):
  """Get the contents of the arp table (arp -a)"""
  listsize = len(hostlist[0])
  cmd = ["/usr/sbin/arp", "-a"]
  arp = sp.Popen(cmd, stdout=sp.PIPE, stderr=sp.PIPE)
  output, err = arp.communicate()
  if DEBUG:
    print err
  if DEBUG:
    print output
  entries = output.splitlines()

  # make a list of the IPs
  column0list = [hostlist[i][0] for i in xrange(len(hostlist))]
  if DEBUG:
    print "\t", column0list

  # Add `arp` data to the array
  for idx, line in enumerate(entries):
    if DEBUG:
      print idx, line
    items = line.split()
    # IP according to arp
    ip = items[1][1:-1]
    try:
      adx = column0list.index(ip)
      # arp hostname
      hostlist[adx][2] = items[0]
      hostlist[adx][8] = int(ip.split('.')[3])
    except ValueError:
      hostlist.extend([[None] * listsize])
      adx = len(hostlist)-1
      hostlist[adx][0] = ip
      hostlist[adx][8] = int(ip.split('.')[3])
      hostlist[adx][1] = items[0]
      hostlist[adx][3] = items[3]
      hostlist[adx][2] = items[0]
      hostlist[adx][9] = -1
      column0list.extend([hostlist[i][0]])

  return hostlist

def pingpong(hostlist):
  """Ping each host in the list and store the timings."""
  for idx, line in enumerate(hostlist):
    ip = line[0]
    pong = map(float, ping(ip, 1))
    if pong[0] > 0:
      pong = map(float, ping(ip, 10))
    hostlist[idx][4] = pong[0]
    hostlist[idx][5] = pong[1]
    hostlist[idx][6] = pong[2]
    hostlist[idx][7] = pong[3]
    if (pong[0] == 0) and (hostlist[idx][9] is None):
        hostlist[idx][9] = 0

  return hostlist

def ping(ip, cnt):
  """Ping the given `ip`  `cnt` times and return responsetimes"""
  cmd = ["ping", "-w", "1", "-q", "-i", "0.5", "-c", str(cnt), ip]
  ping = sp.Popen(cmd, stdout=sp.PIPE, stderr=sp.PIPE)
  output, err = ping.communicate()

  # get last line of output
  line = output.splitlines()[-1]
  # => rtt min/avg/max/mdev = 1.069/1.257/1.777/0.302 ms
  if (len(line) < 12):
    line = 'rtt min/avg/max/mdev = 0.00/0.00/0.00/0.00 ms'

  # get third field
  field3 = line.split()[3]
  # ==> 1.069/1.257/1.777/0.302
  # split the field at "/"
  result = field3.split('/')
  # ===> ['1.036', '1.224', '1.496', '0.171']
  return result

def getkey(item):
  """'row' in list to sort by"""
  return item[8]

def red(text):
  """Format a string to make it show red on an ANSI terminal"""
  print ("\033[91m {}\033[00m" .format(text))

def syslog_trace(trace):
  """Log a python stack trace to syslog"""
  log_lines = trace.split('\n')
  for line in log_lines:
    if line:
      syslog.syslog(syslog.LOG_ALERT, line)

if __name__ == '__main__':
  DEBUG = False
  lsa = len(sys.argv)
  if (lsa == 1):
    sw = 2
  else:
    sw = 2
    if (sys.argv[1] == '-t'):
      sw = 1
    # {endif}
  # {endif}
  try:
    ux = getuxtime()
    ux = map(int, ux)[0]

    #  0 = IP
    #  1 = hostname (dhcp lease)
    #  2 = hostname (arp)
    #  3 = MAC
    #  4 = ping min
    #  5 = ping avg
    #  6 = ping max
    #  7 = ping stdev
    #  8 = IP(...4)
    #  9 = Time to release (minutes)
    # 10 = lastseen
    hostlist = getleases(11)  # parameter is size of the array
    if DEBUG:
      print len(hostlist), "\n"

    hostlist = getarp(hostlist)  # add the hosts that no longer have a lease but are still present in the arp cache
    if DEBUG:
      print len(hostlist), "\n"

    hostlist = sorted(hostlist, key=getkey)  # sort the list by IP octet 4

    hostlist = pingpong(hostlist)  # search for signs of life

    hostlist = lstvssql(hostlist)  # compare the list with the database

    # determine fieldlength of hostname for printing.
    lenhost = 0
    for idx, line in enumerate(hostlist):
      if len(hostlist[idx][1]) > lenhost:
        lenhost = len(hostlist[idx][1])
    # {endfor}

    for idx, line in enumerate(hostlist):
      spc0 = ' ' * (16 - len(line[0]))
      spc1 = ' ' * (lenhost - len(line[1]) + 1)
      spc2 = ' ' * (17 - len(line[3]) + 1)
      if (sw == 1):
        print line[0], spc0, line[1], spc1, line[3], spc2, "avg=", line[5], "\tstdev=", line[7], "\tT2R=", line[9]
      if (sw == 2):
        print line[0], spc0, line[1], spc1, line[3], spc2, "last seen :", line[10]
    # {endfor}

  except Exception as e:
    if DEBUG:
      print("Unexpected error:")
      red(e.message)
    syslog.syslog(syslog.LOG_ALERT, e.__doc__)
    syslog_trace(traceback.format_exc())
    raise
  # {endtry}
<|endoftext|>


<commit_before># TmServer - TissueMAPS server application.
# Copyright (C) 2016  Markus D. Herrmann, University of Zurich and Robin Hafen
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
"""API view functions for querying :mod:`tools <tmlib.tools>` resources
as well as related :mod:`result <tmlib.models.result>` and
:mod:`plot <tmlib.models.plot>` resources.
"""
import os
import json
import logging
from flask import jsonify, request, current_app
from flask_jwt import jwt_required, current_identity
from sqlalchemy import distinct

import tmlib.models as tm
from tmlib import cfg as tmlib_cfg
from tmlib.writers import JsonWriter
from tmlib.tools.jobs import ToolJob
from tmlib.log import LEVELS_TO_VERBOSITY
from tmlib.tools import get_available_tools, get_tool_class
from tmlib.tools.manager import ToolRequestManager

from tmserver.api import api
from tmserver.error import (
    MalformedRequestError,
    ResourceNotFoundError,
    NotAuthorizedError
)
from tmserver.util import decode_query_ids, decode_form_ids
from tmserver.util import assert_query_params, assert_form_params
from tmserver.model import encode_pk
from tmserver.extensions import gc3pie
from tmserver import cfg as server_cfg

logger = logging.getLogger(__name__)

def _create_mapobject_feature(mapobject_id, geometry_description):
    """Creates a GeoJSON feature for the given mapobject and GeoJSON geometry.

    Parameters
    ----------
    mapobject_id: int
        ID of the mapobject
    geometry_description: XXX
        description of a GeoJSON geometry

    Returns
    -------
    dict
    """
    return {
        'type': 'Feature',
        'geometry': geometry_description,
        'properties': {
            'id': str(mapobject_id)
        }
    }

@api.route('/tools', methods=['GET'])
@jwt_required()
def get_tools():
    """
    .. http:get:: /api/tools

        Get a list of supported tools.

        **Example response**:

        .. sourcecode:: http

            HTTP/1.1 200 OK
            Content-Type: application/json

            {
                "data": [
                    {
                        "name": "Cluster Tool",
                        "icon": "CLU",
                        "description",
                        "methods": [...]
                    },
                    ...
                ]
            }

        :reqheader Authorization: JWT token issued by the server
        :statuscode 200: no error

    """
    logger.info('get available tools')
    tool_descriptions = list()
    available_tools = get_available_tools()
    for name in available_tools:
        tool_cls = get_tool_class(name)
        tool_descriptions.append({
            'name': tool_cls.__name__,
            'icon': tool_cls.__icon__,
            'description': tool_cls.__description__,
            'methods': getattr(tool_cls, '__methods__', [])
        })
    return jsonify(data=tool_descriptions)

@api.route(
    '/experiments/<experiment_id>/tools/request', methods=['POST']
)
@jwt_required()
@decode_query_ids('read')
@assert_form_params('payload', 'session_uuid', 'tool_name')
def process_tool_request(experiment_id):
    """
    .. http:post:: /api/experiments/(string:experiment_id)/tools/request

        Processes a generic tool request sent by the client.

        **Example request**:

        .. sourcecode:: http

            Content-Type: application/json

            {
                "tool_name": "Cluster Tool",
                "payload": any object,
                "session_uuid": string
            }

        **Example response**:

        .. sourcecode:: http

            HTTP/1.1 200 OK
            Content-Type: application/json

            {
                "data": {
                    "submission_id": "MQ=="
                }
            }

        :reqheader Authorization: JWT token issued by the server
        :statuscode 200: no error

    """
    data = request.get_json()
    payload = data.get('payload', {})
    session_uuid = data.get('session_uuid')
    tool_name = data.get('tool_name')
    logger.info('process request of tool "%s"', tool_name)

    manager = ToolRequestManager(
        experiment_id, tool_name, server_cfg.logging_verbosity
    )
    submission_id, user_name = manager.register_submission(current_identity.id)
    manager.store_payload(payload, submission_id)
    job = manager.create_job(submission_id, user_name)

    # with tm.utils.ExperimentSession(experiment_id) as session:
    #     session = session.get_or_create(ToolSession, uuid=session_uuid)
    #     session_id = session.id

    gc3pie.store_task(job)
    gc3pie.submit_jobs(job)

    return jsonify(data={
        'submission_id': submission_id
    })

@api.route(
    '/experiments/<experiment_id>/tools/results/<tool_result_id>',
    methods=['GET']
)
@jwt_required()
@decode_query_ids('read')
def get_tool_result(experiment_id, tool_result_id):
    """
    .. http:get:: /api/experiments/(string:experiment_id)/tools/results/(string:tool_result_id)

        Get the result of a previous tool request including a label layer that
        can be queried for tiled cell labels as well as optional plots.

        **Example response**:

        .. sourcecode:: http

            HTTP/1.1 200 OK
            Content-Type: application/json

            {
                "data": {
                    "id": "MQ==",
                    "name": "Cluster Result 1",
                    "submission_id": 117,
                    "layer": {
                        "id": "MQ==",
                        "type": "HeatmapLayer",
                        "attributes": {
                            "min": 0,
                            "max": 2414
                        }
                    },
                    "plots": []
                }
            }

        :statuscode 400: malformed request
        :statuscode 200: no error

    """
    logger.info(
        'get tool result %d for experiment %d', tool_result_id, experiment_id
    )
    with tm.utils.ExperimentSession(experiment_id) as session:
        tool_result = session.query(tm.ToolResult).get(tool_result_id)
        if tool_result is None:
            raise ResourceNotFoundError(tm.ToolResult)
        return jsonify(data=tool_result)

@api.route(
    '/experiments/<experiment_id>/tools/results/<tool_result_id>',
    methods=['PUT']
)
@jwt_required()
@decode_query_ids('read')
def update_tool_result(experiment_id, tool_result_id):
    """
    .. http:put:: /api/experiments/(string:experiment_id)/tools/result/(string:tool_result_id)

        Update a :class:`ToolResult <tmlib.models.result.ToolResult>`.

        **Example request**:

        .. sourcecode:: http

            Content-Type: application/json

            {
                "name": "New Name"
            }

        **Example response**:

        .. sourcecode:: http

            HTTP/1.1 200 OK
            Content-Type: application/json

            {
                "message": "ok"
            }

        :statuscode 400: malformed request
        :statuscode 200: no error

    """
    data = request.get_json()
    name = data.get('name')
    logger.info(
        'rename tool result %d of experiment %d', tool_result_id, experiment_id
    )
    with tm.utils.ExperimentSession(experiment_id) as session:
        tool_result = session.query(tm.ToolResult).get(tool_result_id)
        tool_result.name = name
    return jsonify(message='ok')

@api.route(
    '/experiments/<experiment_id>/tools/results', methods=['GET']
)
@jwt_required()
@decode_query_ids('read')
def get_tool_results(experiment_id):
    """
    .. http:get:: /api/experiments/(string:experiment_id)/tools/result

        Get the result of a previous tool request including a label layer that
        can be queried for tiled cell labels as well as optional plots.

        **Example request**:

        .. sourcecode:: http

            Content-Type: application/json

            {
                "submission_id": 117
            }

        **Example response**:

        .. sourcecode:: http

            HTTP/1.1 200 OK
            Content-Type: application/json

            {
                "data": {
                    "id": "MQ==",
                    "name": "Cluster Result 1",
                    "submission_id": 117,
                    "layer": {
                        "id": "MQ==",
                        "type": "HeatmapLayer",
                        "attributes": {
                            "min": 0,
                            "max": 2414
                        }
                    },
                    "plots": []
                }
            }

        :query submission_id: ID of the corresponding submission (optional)

        :statuscode 400: malformed request
        :statuscode 200: no error

    """
    submission_id = request.args.get('submission_id', type=int)
    logger.info('get tool results')

    if submission_id is not None:
        logger.info(
            'filter tool results for submissions %d', submission_id
        )
        submission_ids = [submission_id]
    else:
        with tm.utils.MainSession() as session:
            logger.debug('filter tool results for current user')
            submissions = session.query(tm.Submission.id).\
                filter_by(user_id=current_identity.id, program='tool').\
                all()
            submission_ids = [s.id for s in submissions]
    with tm.utils.ExperimentSession(experiment_id) as session:
        tool_results = session.query(tm.ToolResult).\
            filter(tm.ToolResult.submission_id.in_(submission_ids)).\
            order_by(tm.ToolResult.submission_id).\
            distinct(tm.ToolResult.submission_id).\
            all()
        return jsonify(data=tool_results)

@api.route(
    '/experiments/<experiment_id>/tools/results/<tool_result_id>',
    methods=['DELETE']
)
@jwt_required()
@decode_query_ids('read')
def delete_tool_result(experiment_id, tool_result_id):
    """
    .. http:delete:: /api/experiments/(string:experiment_id)/tools/result/(string:tool_result_id)

        Delete a tool result.

        **Example response**:

        .. sourcecode:: http

            HTTP/1.1 200 OK
            Content-Type: application/json

            {
                "message": "ok"
            }

        :statuscode 200: no error

    """
    logger.info('delete tool result %d', tool_result_id)
    with tm.utils.ExperimentSession(experiment_id) as session:
        session.query(tm.ToolResult).\
            filter_by(id=tool_result_id).\
            delete()
    return jsonify(message='ok')

@api.route(
    '/experiments/<experiment_id>/tools/status', methods=['GET']
)
@jwt_required()
@decode_query_ids('read')
def get_tool_job_status(experiment_id):
    """
    .. http:get:: /api/experiments/(string:experiment_id)/tools/status

        Get the status of one or multiple jobs processing a tool request.

        **Example response**:

        .. sourcecode:: http

            HTTP/1.1 200 OK
            Content-Type: application/json

            {
                "data": [
                    {
                        "state": string,
                        "submission_id": number,
                        "exitcode": number
                    },
                    ...
                ]
            }

        :query submission_id: numeric ID of the submission for which the job status should be retrieved (optional)
        :query state: state jobs should have, e.g. RUNNING (optional)
        :statuscode 400: malformed request
        :statuscode 200: no error

    """
    logger.info('get status of tool jobs for experiment %d', experiment_id)
    submission_id = request.args.get('submission_id', type=int)
    state = request.args.get('state')

    with tm.utils.MainSession() as session:
        tool_jobs = session.query(
                tm.Task.state, tm.Task.submission_id, tm.Task.exitcode
            ).\
            join(tm.Submission).\
            filter(
                tm.Submission.program == 'tool',
                tm.Submission.experiment_id == experiment_id,
                tm.Submission.user_id == current_identity.id
            )
        if state is not None:
            logger.info('filter tool jobs for state "%s"', state)
            tool_jobs = tool_jobs.\
                filter(tm.Task.state == state)
        if submission_id is not None:
            logger.info('filter tool jobs for submission %d', submission_id)
            tool_jobs = tool_jobs.\
                filter(tm.Task.submission_id == submission_id)
        tool_jobs = tool_jobs.all()
        tool_job_status = [
            {
                'state': j.state,
                'submission_id': j.submission_id,
                'exitcode': j.exitcode
            }
            for j in tool_jobs
        ]
        return jsonify(data=tool_job_status)
<commit_msg>Fix bug in api tools module<commit_after># TmServer - TissueMAPS server application.
# Copyright (C) 2016  Markus D. Herrmann, University of Zurich and Robin Hafen
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
"""API view functions for querying :mod:`tools <tmlib.tools>` resources
as well as related :mod:`result <tmlib.models.result>` and
:mod:`plot <tmlib.models.plot>` resources.
"""
import os
import json
import logging
from flask import jsonify, request, current_app
from flask_jwt import jwt_required, current_identity
from sqlalchemy import distinct

import tmlib.models as tm
from tmlib import cfg as tmlib_cfg
from tmlib.writers import JsonWriter
from tmlib.tools.jobs import ToolJob
from tmlib.log import LEVELS_TO_VERBOSITY
from tmlib.tools import get_available_tools, get_tool_class
from tmlib.tools.manager import ToolRequestManager

from tmserver.api import api
from tmserver.error import (
    MalformedRequestError,
    ResourceNotFoundError,
    NotAuthorizedError
)
from tmserver.util import decode_query_ids, decode_form_ids
from tmserver.util import assert_query_params, assert_form_params
from tmserver.model import encode_pk
from tmserver.extensions import gc3pie
from tmserver import cfg as server_cfg

logger = logging.getLogger(__name__)

def _create_mapobject_feature(mapobject_id, geometry_description):
    """Creates a GeoJSON feature for the given mapobject and GeoJSON geometry.

    Parameters
    ----------
    mapobject_id: int
        ID of the mapobject
    geometry_description: XXX
        description of a GeoJSON geometry

    Returns
    -------
    dict
    """
    return {
        'type': 'Feature',
        'geometry': geometry_description,
        'properties': {
            'id': str(mapobject_id)
        }
    }

@api.route('/tools', methods=['GET'])
@jwt_required()
def get_tools():
    """
    .. http:get:: /api/tools

        Get a list of supported tools.

        **Example response**:

        .. sourcecode:: http

            HTTP/1.1 200 OK
            Content-Type: application/json

            {
                "data": [
                    {
                        "name": "Cluster Tool",
                        "icon": "CLU",
                        "description",
                        "methods": [...]
                    },
                    ...
                ]
            }

        :reqheader Authorization: JWT token issued by the server
        :statuscode 200: no error

    """
    logger.info('get available tools')
    tool_descriptions = list()
    available_tools = get_available_tools()
    for name in available_tools:
        tool_cls = get_tool_class(name)
        tool_descriptions.append({
            'name': tool_cls.__name__,
            'icon': tool_cls.__icon__,
            'description': tool_cls.__description__,
            'methods': getattr(tool_cls, '__methods__', [])
        })
    return jsonify(data=tool_descriptions)

@api.route(
    '/experiments/<experiment_id>/tools/request', methods=['POST']
)
@jwt_required()
@decode_query_ids('read')
@assert_form_params('payload', 'session_uuid', 'tool_name')
def process_tool_request(experiment_id):
    """
    .. http:post:: /api/experiments/(string:experiment_id)/tools/request

        Processes a generic tool request sent by the client.

        **Example request**:

        .. sourcecode:: http

            Content-Type: application/json

            {
                "tool_name": "Cluster Tool",
                "payload": any object,
                "session_uuid": string
            }

        **Example response**:

        .. sourcecode:: http

            HTTP/1.1 200 OK
            Content-Type: application/json

            {
                "data": {
                    "submission_id": "MQ=="
                }
            }

        :reqheader Authorization: JWT token issued by the server
        :statuscode 200: no error

    """
    data = request.get_json()
    payload = data.get('payload', {})
    session_uuid = data.get('session_uuid')
    tool_name = data.get('tool_name')
    logger.info('process request of tool "%s"', tool_name)

    manager = ToolRequestManager(
        experiment_id, tool_name, server_cfg.logging_verbosity
    )
    submission_id, user_name = manager.register_submission(current_identity.id)
    manager.store_payload(payload, submission_id)
    job = manager.create_job(submission_id, user_name)

    # with tm.utils.ExperimentSession(experiment_id) as session:
    #     session = session.get_or_create(ToolSession, uuid=session_uuid)
    #     session_id = session.id

    gc3pie.store_task(job)
    gc3pie.submit_task(job)

    return jsonify(data={
        'submission_id': submission_id
    })

@api.route(
    '/experiments/<experiment_id>/tools/results/<tool_result_id>',
    methods=['GET']
)
@jwt_required()
@decode_query_ids('read')
def get_tool_result(experiment_id, tool_result_id):
    """
    .. http:get:: /api/experiments/(string:experiment_id)/tools/results/(string:tool_result_id)

        Get the result of a previous tool request including a label layer that
        can be queried for tiled cell labels as well as optional plots.

        **Example response**:

        .. sourcecode:: http

            HTTP/1.1 200 OK
            Content-Type: application/json

            {
                "data": {
                    "id": "MQ==",
                    "name": "Cluster Result 1",
                    "submission_id": 117,
                    "layer": {
                        "id": "MQ==",
                        "type": "HeatmapLayer",
                        "attributes": {
                            "min": 0,
                            "max": 2414
                        }
                    },
                    "plots": []
                }
            }

        :statuscode 400: malformed request
        :statuscode 200: no error

    """
    logger.info(
        'get tool result %d for experiment %d', tool_result_id, experiment_id
    )
    with tm.utils.ExperimentSession(experiment_id) as session:
        tool_result = session.query(tm.ToolResult).get(tool_result_id)
        if tool_result is None:
            raise ResourceNotFoundError(tm.ToolResult)
        return jsonify(data=tool_result)

@api.route(
    '/experiments/<experiment_id>/tools/results/<tool_result_id>',
    methods=['PUT']
)
@jwt_required()
@decode_query_ids('read')
def update_tool_result(experiment_id, tool_result_id):
    """
    .. http:put:: /api/experiments/(string:experiment_id)/tools/result/(string:tool_result_id)

        Update a :class:`ToolResult <tmlib.models.result.ToolResult>`.

        **Example request**:

        .. sourcecode:: http

            Content-Type: application/json

            {
                "name": "New Name"
            }

        **Example response**:

        .. sourcecode:: http

            HTTP/1.1 200 OK
            Content-Type: application/json

            {
                "message": "ok"
            }

        :statuscode 400: malformed request
        :statuscode 200: no error

    """
    data = request.get_json()
    name = data.get('name')
    logger.info(
        'rename tool result %d of experiment %d', tool_result_id, experiment_id
    )
    with tm.utils.ExperimentSession(experiment_id) as session:
        tool_result = session.query(tm.ToolResult).get(tool_result_id)
        tool_result.name = name
    return jsonify(message='ok')

@api.route(
    '/experiments/<experiment_id>/tools/results', methods=['GET']
)
@jwt_required()
@decode_query_ids('read')
def get_tool_results(experiment_id):
    """
    .. http:get:: /api/experiments/(string:experiment_id)/tools/result

        Get the result of a previous tool request including a label layer that
        can be queried for tiled cell labels as well as optional plots.

        **Example request**:

        .. sourcecode:: http

            Content-Type: application/json

            {
                "submission_id": 117
            }

        **Example response**:

        .. sourcecode:: http

            HTTP/1.1 200 OK
            Content-Type: application/json

            {
                "data": {
                    "id": "MQ==",
                    "name": "Cluster Result 1",
                    "submission_id": 117,
                    "layer": {
                        "id": "MQ==",
                        "type": "HeatmapLayer",
                        "attributes": {
                            "min": 0,
                            "max": 2414
                        }
                    },
                    "plots": []
                }
            }

        :query submission_id: ID of the corresponding submission (optional)

        :statuscode 400: malformed request
        :statuscode 200: no error

    """
    submission_id = request.args.get('submission_id', type=int)
    logger.info('get tool results')

    if submission_id is not None:
        logger.info(
            'filter tool results for submissions %d', submission_id
        )
        submission_ids = [submission_id]
    else:
        with tm.utils.MainSession() as session:
            logger.debug('filter tool results for current user')
            submissions = session.query(tm.Submission.id).\
                filter_by(user_id=current_identity.id, program='tool').\
                all()
            submission_ids = [s.id for s in submissions]
    with tm.utils.ExperimentSession(experiment_id) as session:
        tool_results = session.query(tm.ToolResult).\
            filter(tm.ToolResult.submission_id.in_(submission_ids)).\
            order_by(tm.ToolResult.submission_id).\
            distinct(tm.ToolResult.submission_id).\
            all()
        return jsonify(data=tool_results)

@api.route(
    '/experiments/<experiment_id>/tools/results/<tool_result_id>',
    methods=['DELETE']
)
@jwt_required()
@decode_query_ids('read')
def delete_tool_result(experiment_id, tool_result_id):
    """
    .. http:delete:: /api/experiments/(string:experiment_id)/tools/result/(string:tool_result_id)

        Delete a tool result.

        **Example response**:

        .. sourcecode:: http

            HTTP/1.1 200 OK
            Content-Type: application/json

            {
                "message": "ok"
            }

        :statuscode 200: no error

    """
    logger.info('delete tool result %d', tool_result_id)
    with tm.utils.ExperimentSession(experiment_id) as session:
        session.query(tm.ToolResult).\
            filter_by(id=tool_result_id).\
            delete()
    return jsonify(message='ok')

@api.route(
    '/experiments/<experiment_id>/tools/status', methods=['GET']
)
@jwt_required()
@decode_query_ids('read')
def get_tool_job_status(experiment_id):
    """
    .. http:get:: /api/experiments/(string:experiment_id)/tools/status

        Get the status of one or multiple jobs processing a tool request.

        **Example response**:

        .. sourcecode:: http

            HTTP/1.1 200 OK
            Content-Type: application/json

            {
                "data": [
                    {
                        "state": string,
                        "submission_id": number,
                        "exitcode": number
                    },
                    ...
                ]
            }

        :query submission_id: numeric ID of the submission for which the job status should be retrieved (optional)
        :query state: state jobs should have, e.g. RUNNING (optional)
        :statuscode 400: malformed request
        :statuscode 200: no error

    """
    logger.info('get status of tool jobs for experiment %d', experiment_id)
    submission_id = request.args.get('submission_id', type=int)
    state = request.args.get('state')

    with tm.utils.MainSession() as session:
        tool_jobs = session.query(
                tm.Task.state, tm.Task.submission_id, tm.Task.exitcode
            ).\
            join(tm.Submission).\
            filter(
                tm.Submission.program == 'tool',
                tm.Submission.experiment_id == experiment_id,
                tm.Submission.user_id == current_identity.id
            )
        if state is not None:
            logger.info('filter tool jobs for state "%s"', state)
            tool_jobs = tool_jobs.\
                filter(tm.Task.state == state)
        if submission_id is not None:
            logger.info('filter tool jobs for submission %d', submission_id)
            tool_jobs = tool_jobs.\
                filter(tm.Task.submission_id == submission_id)
        tool_jobs = tool_jobs.all()
        tool_job_status = [
            {
                'state': j.state,
                'submission_id': j.submission_id,
                'exitcode': j.exitcode
            }
            for j in tool_jobs
        ]
        return jsonify(data=tool_job_status)
<|endoftext|>