<commit_before>"""
Generic utility routines for number handling and calculating (specific)
variances used by the TKP sourcefinder.
"""

import numpy
from numpy.ma import MaskedArray
from scipy.special import erf
from scipy.special import erfcinv
from scipy.optimize import fsolve

from .utils import calculate_correlation_lengths

# CODE & NUMBER HANDLING ROUTINES
#
def var_helper(N):
    """Correct for the fact the rms noise is computed from a clipped
    distribution.

    That noise will always be lower than the noise from the complete
    distribution.  The correction factor is a function of the computed
    rms noise only.
    """
    term1 = numpy.sqrt(2. * numpy.pi) * erf(N / numpy.sqrt(2.))
    term2 = 2. * N * numpy.exp(-N ** 2 / 2.)
    return term1 / (term1 - term2)

def find_true_std(sigma, clip_limit, clipped_std):
    help1 = clip_limit/(sigma*numpy.sqrt(2))
    help2 = numpy.sqrt(2*numpy.pi)*erf(help1)
    return sigma**2*(help2-2*numpy.sqrt(2)*help1*numpy.exp(-help1**2))-clipped_std**2*help2

def indep_pixels(N, beam):
    corlengthlong, corlengthshort = calculate_correlation_lengths(
        beam[0], beam[1])
    correlated_area = 0.25 * numpy.pi * corlengthlong * corlengthshort
    return N / correlated_area

def sigma_clip(data, beam, kappa=2.0, max_iter=100,
               centref=numpy.median, distf=numpy.var, my_iterations=0):
    """Iterative clipping

    By default, this performs clipping of the standard deviation about the
    median of the data. But by tweaking centref/distf, it could be much
    more general.

    max_iter sets the maximum number of iterations used.

    my_iterations is a counter for recursive operation of the code; leave it
    alone unless you really want to pretend to jump into the middle of a loop.

    sigma is subtle: if a callable is given, it is passed a copy of the data
    array and can calculate a clipping limit. See, for e.g., unbiased_sigma()
    defined above. However, if it isn't callable, sigma is assumed to just set
    a hard limit.

    To do: Improve documentation
            -Returns???
            -How does it make use of the beam? (It estimates the noise correlation)
    """
    if my_iterations >= max_iter:
        # Exceeded maximum number of iterations; return
        return data, my_iterations

    # Numpy 1.1 breaks std() for MaskedArray: see
    # <http://www.scipy.org/scipy/numpy/wiki/MaskedArray>.
    # MaskedArray.compressed() returns a 1-D array of non-masked data.
    if isinstance(data, MaskedArray):
        data = data.compressed()
    centre = centref(data)
    N = numpy.size(data)
    N_indep = indep_pixels(N, beam)
    if N_indep < 1:
        # This chunk is too small for processing; return an empty array.
        return numpy.array([]), 0, 0, 0

    # distf=numpy.var is a sample variance with the factor N/(N-1)
    # already built in, N being the number of pixels. So, we are
    # going to remove that and replace it by N_indep/(N_indep-1)
    clipped_var = distf(data) * (N - 1.) * N_indep / (N * (N_indep - 1.))
    # unbiased_var = corr_clip * clipped_var

    # There is an extra factor c4 needed to get a unbiased standard
    # deviation, unbiased if we disregard clipping bias, see
    # http://en.wikipedia.org/wiki/Unbiased_estimation_of_standard_deviation\
    #         #Results_for_the_normal_distribution
    c4 = 1. - 0.25 / N_indep - 0.21875 / N_indep ** 2
    std_corr_for_limited_sample_size = numpy.sqrt(clipped_var) / c4
    limit = kappa * std_corr_for_limited_sample_size
    unbiased_std = fsolve(find_true_std, std_corr_for_limited_sample_size,
                          args=(limit, std_corr_for_limited_sample_size))[0]

    newdata = data.compress(abs(data - centre) <= limit)

    if len(newdata) != len(data) and len(newdata) > 0:
        my_iterations += 1
        return sigma_clip(newdata, beam, kappa, max_iter, centref, distf,
                          my_iterations)
    else:
        return newdata, unbiased_std, centre, my_iterations
<commit_msg>No need to computed the std, corrected for clipping bias, after every clip.<commit_after>"""
Generic utility routines for number handling and calculating (specific)
variances used by the TKP sourcefinder.
"""

import numpy
from numpy.ma import MaskedArray
from scipy.special import erf
from scipy.special import erfcinv
from scipy.optimize import fsolve

from .utils import calculate_correlation_lengths

# CODE & NUMBER HANDLING ROUTINES
#
def var_helper(N):
    """Correct for the fact the rms noise is computed from a clipped
    distribution.

    That noise will always be lower than the noise from the complete
    distribution.  The correction factor is a function of the computed
    rms noise only.
    """
    term1 = numpy.sqrt(2. * numpy.pi) * erf(N / numpy.sqrt(2.))
    term2 = 2. * N * numpy.exp(-N ** 2 / 2.)
    return term1 / (term1 - term2)

def find_true_std(sigma, clip_limit, clipped_std):
    help1 = clip_limit/(sigma*numpy.sqrt(2))
    help2 = numpy.sqrt(2*numpy.pi)*erf(help1)
    return sigma**2*(help2-2*numpy.sqrt(2)*help1*numpy.exp(-help1**2))-clipped_std**2*help2

def indep_pixels(N, beam):
    corlengthlong, corlengthshort = calculate_correlation_lengths(
        beam[0], beam[1])
    correlated_area = 0.25 * numpy.pi * corlengthlong * corlengthshort
    return N / correlated_area

def sigma_clip(data, beam, kappa=2.0, max_iter=100,
               centref=numpy.median, distf=numpy.var, my_iterations=0):
    """Iterative clipping

    By default, this performs clipping of the standard deviation about the
    median of the data. But by tweaking centref/distf, it could be much
    more general.

    max_iter sets the maximum number of iterations used.

    my_iterations is a counter for recursive operation of the code; leave it
    alone unless you really want to pretend to jump into the middle of a loop.

    sigma is subtle: if a callable is given, it is passed a copy of the data
    array and can calculate a clipping limit. See, for e.g., unbiased_sigma()
    defined above. However, if it isn't callable, sigma is assumed to just set
    a hard limit.

    To do: Improve documentation
            -Returns???
            -How does it make use of the beam? (It estimates the noise correlation)
    """
    if my_iterations >= max_iter:
        # Exceeded maximum number of iterations; return
        return data, my_iterations

    # Numpy 1.1 breaks std() for MaskedArray: see
    # <http://www.scipy.org/scipy/numpy/wiki/MaskedArray>.
    # MaskedArray.compressed() returns a 1-D array of non-masked data.
    if isinstance(data, MaskedArray):
        data = data.compressed()
    centre = centref(data)
    N = numpy.size(data)
    N_indep = indep_pixels(N, beam)
    if N_indep < 1:
        # This chunk is too small for processing; return an empty array.
        return numpy.array([]), 0, 0, 0

    # distf=numpy.var is a sample variance with the factor N/(N-1)
    # already built in, N being the number of pixels. So, we are
    # going to remove that and replace it by N_indep/(N_indep-1)
    clipped_var = distf(data) * (N - 1.) * N_indep / (N * (N_indep - 1.))
    # unbiased_var = corr_clip * clipped_var

    # There is an extra factor c4 needed to get a unbiased standard
    # deviation, unbiased if we disregard clipping bias, see
    # http://en.wikipedia.org/wiki/Unbiased_estimation_of_standard_deviation\
    #         #Results_for_the_normal_distribution
    c4 = 1. - 0.25 / N_indep - 0.21875 / N_indep ** 2
    std_corr_for_limited_sample_size = numpy.sqrt(clipped_var) / c4
    limit = kappa * std_corr_for_limited_sample_size

    newdata = data.compress(abs(data - centre) <= limit)

    if len(newdata) != len(data) and len(newdata) > 0:
        my_iterations += 1
        return sigma_clip(newdata, beam, kappa, max_iter, centref, distf,
                          my_iterations)
    else:
        unbiased_std = fsolve(find_true_std, std_corr_for_limited_sample_size,
                              args=(limit, std_corr_for_limited_sample_size))[0]
        return newdata, unbiased_std, centre, my_iterations
<|endoftext|>


<commit_before>from chromatica import logger
from chromatica.util import load_external_module

import chromatica.util as util

load_external_module(__file__, "")
from clang import cindex

import os
import re

log = logger.logging.getLogger("chromatica.compile_args")

DEFAULT_STD={"c"   : ["-std=c11"], \
             "cpp" : ["-std=c++14"]}

SOURCE_EXTS=[".c", ".cc", ".cpp", ".cxx"]
compile_args_files = ['.color_coded', '.clang', 'compile_flags.txt', '.cquery', '.ccls', '.chromatica', 'compile_commands.json']

def set_default_std(stds):
    DEFAULT_STD = stds
    return True

class CompileArgsDatabase(object):

    def __init__(self, global_path, global_args=None):
        if global_path:
            self.__global_path = global_path

        self.__paths = []
        self.__args_file = None
        self.__cdb_file = None
        self.cdb = None

        self.global_args = global_args
        self.compile_args = []
        self.ft_compile_args = { "c" : [], "cpp" : [] , "objc" : [], "objcpp" : [] }

        self.find_per_project_file()

        if len(self.__paths) > 0:
            self.__args_file = self.__paths[0]

        self.parse_args_file()

    def find_per_project_file(self):
        search_path = os.getcwd()
        found_project_root = False
        while not found_project_root and os.path.dirname(search_path) != search_path:
            for args_file in compile_args_files:
                args_file_path = os.path.join(search_path, args_file)
                if os.path.exists(args_file_path):
                    self.__paths.append(args_file_path)
                    found_project_root = True
            search_path = os.path.dirname(search_path)

    def parse_args_file(self):
        if not self.__args_file:
            return

        filename = os.path.basename(self.__args_file)
        if filename == ".chromatica":
            self.parse_chromatica_file()
        elif filename == ".color_coded" or filename == ".clang" or filename == "compile_flags.txt":
            self.parse_simple_file()
        elif filename == ".cquery":
            self.parse_ccls_file()
        elif filename == ".ccls":
            self.parse_ccls_file()
        elif filename == ".ycm_extra_flags":
            self.parse_ycm_file()
        else:
            self.init_cdb(os.path.dirname(self.__args_file))

    def parse_simple_file(self):
        if self.__args_file == None:
            return
        fp = open(self.__args_file, "r")
        lines = fp.readlines()
        fp.close()

        for line in lines:
            if len(line) == 0:
                continue
            self.compile_args.extend(line.strip().split())

    def parse_chromatica_file(self):
        assert len(self.compile_args) == 0
        if self.__args_file == None:
            return
        fp = open(self.__args_file, "r")
        lines = fp.readlines()
        fp.close()

        funcs = {"flags" : lambda s, value: s.compile_args.extend(value.split()),
                 "c" : lambda s, value: s.ft_compile_args["c"].extend(value.split()),
                 "cpp" : lambda s, value: s.ft_compile_args["cpp"].extend(value.split()),
                 "compilation_database" : lambda s, value: s.init_cdb(value), }

        for line in lines:
            if len(line) == 0 :
                continue
            line = line.strip()
            if line.startswith("#"):
                continue
            pos = line.find("=")
            if pos != -1:
                key = line[:pos]
                try:
                    funcs[key](self, line[pos+1:])
                except:
                    log.error("Invalid configuration key: ", key)
            else:
                self.compile_args.extend(line.split())

    def parse_ccls_file(self):
        if self.__args_file == None:
            return
        fp = open(self.__args_file, "r")
        lines = fp.readlines()
        fp.close()

        for line in lines:
            if len(line) == 0 :
                continue
            line = line.strip()
            if line.startswith("#"):
                continue
            elif line.startswith("%"):
                keys = [key for key in line.split(" ") if len(key) > 0]
                for key in keys:
                    if key == "%compile_commands.json":
                        self.init_cdb("compile_commands.json")
                    elif key == "%c" or key == "%h":
                        self.ft_compile_args["c"].append(keys[-1])
                    elif key == "%cpp" or key == "%hpp":
                        self.ft_compile_args["c"].append(keys[-1])
                    elif key == "%objective-c":
                        self.ft_compile_args["objc"].append(keys[-1])
                    elif key == "%objective-cpp":
                        self.ft_compile_args["objcpp"].append(keys[-1])
            else:
                self.compile_args.append(line)

    def init_cdb(self, value):
        log.info("cdb: %s" % value)
        cdb_rel_path = value.strip("\"")
        if os.path.isabs(cdb_rel_path):
            cdb_path = cdb_rel_path
        else:
            cdb_path = os.path.join(os.path.dirname(self.__args_file), cdb_rel_path)
        if cdb_path and os.path.isdir(cdb_path):
            self.__cdb_path = cdb_path
            # self.try_init_cdb()
            try:
                self.cdb = cindex.CompilationDatabase.fromDirectory(self.__cdb_path)
            except:
                log.error("Invalid compilation database file %s" % self.__cdb_path)
                self.__cdb_path = None

    def get_cdb_args(self, filename):
        res = []
        ret = self.cdb.getCompileCommands(filename)
        _basename = os.path.basename(filename)
        log.info("Read cdb for: %s" % filename)
        if ret:
            for cmds in ret:
                cwd = cmds.directory
                skip = 0
                last = ''
                for arg in cmds.arguments:
                    if skip and (len(arg) == 0 or arg[0] != "-"):
                        skip = 0
                        continue
                    if arg == "-o" or arg == "-c":
                        skip = 1
                        continue

                    if arg != '-I' and arg.startswith('-I'):
                        include_path = arg[2:]
                        if not os.path.isabs(include_path):
                            include_path = os.path.normpath(
                                os.path.join(cwd, include_path))
                        res.append('-I' + include_path)
                    if arg != '-isystem' and arg.startswith('-isystem'):
                        include_path = arg[8:]
                        if not os.path.isabs(include_path):
                            include_path = os.path.normpath(
                                os.path.join(cwd, include_path))
                        res.append('-isystem' + include_path)
                    if _basename in arg:
                        continue;
                    else:
                        # if last added switch was standalone include then we need to append path to it
                        if last == '-I' or last == '-isystem':
                            include_path = arg
                            if not os.path.isabs(include_path):
                                include_path = os.path.normpath(os.path.join(cwd, include_path))
                            res[len(res) - 1] += include_path
                            last = ''
                        else:
                            res.append(arg)
                            last = arg
        else:
            print("Cannot find compile flags for %s in compilation database" % filename)
        return res

    def get_args_filename(self, filename, search_source_args=False):
        ret = []
        if self.global_args != None:
            ret = self.global_args

        ret += self.compile_args

        if self.cdb != None:
            cdb_ret = self.get_cdb_args(filename)
            if not cdb_ret and search_source_args:
                filename_base = os.path.splitext(filename)[0]
                for source_ext in SOURCE_EXTS:
                    cdb_ret = self.get_cdb_args(filename_base + source_ext)
                    if cdb_ret:
                        break
            ret += cdb_ret

        return ret

    def get_args_filename_ft(self, filename, filetype, search_source_args=False):
        ret = self.get_args_filename(filename, search_source_args)
        return ret + self.ft_compile_args[filetype]

    def get_available_args_files(self):
        return self.__paths

    @property
    def args_file(self):
        return self.__args_file
<commit_msg>fix parsing ccls file<commit_after>from chromatica import logger
from chromatica.util import load_external_module

import chromatica.util as util

load_external_module(__file__, "")
from clang import cindex

import os
import re

log = logger.logging.getLogger("chromatica.compile_args")

DEFAULT_STD={"c"   : ["-std=c11"], \
             "cpp" : ["-std=c++14"]}

SOURCE_EXTS=[".c", ".cc", ".cpp", ".cxx"]
compile_args_files = ['.color_coded', '.clang', 'compile_flags.txt', '.cquery', '.ccls', '.chromatica', 'compile_commands.json']

def set_default_std(stds):
    DEFAULT_STD = stds
    return True

class CompileArgsDatabase(object):

    def __init__(self, global_path, global_args=None):
        if global_path:
            self.__global_path = global_path

        self.__paths = []
        self.__args_file = None
        self.__cdb_file = None
        self.cdb = None

        self.global_args = global_args
        self.compile_args = []
        self.ft_compile_args = { "c" : [], "cpp" : [] , "objc" : [], "objcpp" : [] }

        self.find_per_project_file()

        if len(self.__paths) > 0:
            self.__args_file = self.__paths[0]

        self.parse_args_file()

    def find_per_project_file(self):
        search_path = os.getcwd()
        found_project_root = False
        while not found_project_root and os.path.dirname(search_path) != search_path:
            for args_file in compile_args_files:
                args_file_path = os.path.join(search_path, args_file)
                if os.path.exists(args_file_path):
                    self.__paths.append(args_file_path)
                    found_project_root = True
            search_path = os.path.dirname(search_path)

    def parse_args_file(self):
        if not self.__args_file:
            return

        filename = os.path.basename(self.__args_file)
        if filename == ".chromatica":
            self.parse_chromatica_file()
        elif filename == ".color_coded" or filename == ".clang" or filename == "compile_flags.txt":
            self.parse_simple_file()
        elif filename == ".cquery":
            self.parse_ccls_file()
        elif filename == ".ccls":
            self.parse_ccls_file()
        elif filename == ".ycm_extra_flags":
            self.parse_ycm_file()
        else:
            self.init_cdb(os.path.dirname(self.__args_file))

    def parse_simple_file(self):
        if self.__args_file == None:
            return
        fp = open(self.__args_file, "r")
        lines = fp.readlines()
        fp.close()

        for line in lines:
            if len(line) == 0:
                continue
            self.compile_args.extend(line.strip().split())

    def parse_chromatica_file(self):
        assert len(self.compile_args) == 0
        if self.__args_file == None:
            return
        fp = open(self.__args_file, "r")
        lines = fp.readlines()
        fp.close()

        funcs = {"flags" : lambda s, value: s.compile_args.extend(value.split()),
                 "c" : lambda s, value: s.ft_compile_args["c"].extend(value.split()),
                 "cpp" : lambda s, value: s.ft_compile_args["cpp"].extend(value.split()),
                 "compilation_database" : lambda s, value: s.init_cdb(value), }

        for line in lines:
            if len(line) == 0 :
                continue
            line = line.strip()
            if line.startswith("#"):
                continue
            pos = line.find("=")
            if pos != -1:
                key = line[:pos]
                try:
                    funcs[key](self, line[pos+1:])
                except:
                    log.error("Invalid configuration key: ", key)
            else:
                self.compile_args.extend(line.split())

    def parse_ccls_file(self):
        if self.__args_file == None:
            return
        fp = open(self.__args_file, "r")
        lines = fp.readlines()
        fp.close()

        for line in lines:
            if len(line) == 0 :
                continue
            line = line.strip()
            if line.startswith("clang"):
                continue
            if line.startswith("#"):
                continue
            elif line.startswith("%"):
                keys = [key for key in line.split(" ") if len(key) > 0]
                for key in keys:
                    if key == "%compile_commands.json":
                        self.init_cdb("compile_commands.json")
                    elif key == "%c" or key == "%h":
                        self.ft_compile_args["c"].append(keys[-1])
                    elif key == "%cpp" or key == "%hpp":
                        self.ft_compile_args["c"].append(keys[-1])
                    elif key == "%objective-c":
                        self.ft_compile_args["objc"].append(keys[-1])
                    elif key == "%objective-cpp":
                        self.ft_compile_args["objcpp"].append(keys[-1])
            else:
                self.compile_args.append(line)

    def init_cdb(self, value):
        log.info("cdb: %s" % value)
        cdb_rel_path = value.strip("\"")
        if os.path.isabs(cdb_rel_path):
            cdb_path = cdb_rel_path
        else:
            cdb_path = os.path.join(os.path.dirname(self.__args_file), cdb_rel_path)
        if cdb_path and os.path.isdir(cdb_path):
            self.__cdb_path = cdb_path
            # self.try_init_cdb()
            try:
                self.cdb = cindex.CompilationDatabase.fromDirectory(self.__cdb_path)
            except:
                log.error("Invalid compilation database file %s" % self.__cdb_path)
                self.__cdb_path = None

    def get_cdb_args(self, filename):
        res = []
        ret = self.cdb.getCompileCommands(filename)
        _basename = os.path.basename(filename)
        log.info("Read cdb for: %s" % filename)
        if ret:
            for cmds in ret:
                cwd = cmds.directory
                skip = 0
                last = ''
                for arg in cmds.arguments:
                    if skip and (len(arg) == 0 or arg[0] != "-"):
                        skip = 0
                        continue
                    if arg == "-o" or arg == "-c":
                        skip = 1
                        continue

                    if arg != '-I' and arg.startswith('-I'):
                        include_path = arg[2:]
                        if not os.path.isabs(include_path):
                            include_path = os.path.normpath(
                                os.path.join(cwd, include_path))
                        res.append('-I' + include_path)
                    if arg != '-isystem' and arg.startswith('-isystem'):
                        include_path = arg[8:]
                        if not os.path.isabs(include_path):
                            include_path = os.path.normpath(
                                os.path.join(cwd, include_path))
                        res.append('-isystem' + include_path)
                    if _basename in arg:
                        continue;
                    else:
                        # if last added switch was standalone include then we need to append path to it
                        if last == '-I' or last == '-isystem':
                            include_path = arg
                            if not os.path.isabs(include_path):
                                include_path = os.path.normpath(os.path.join(cwd, include_path))
                            res[len(res) - 1] += include_path
                            last = ''
                        else:
                            res.append(arg)
                            last = arg
        else:
            print("Cannot find compile flags for %s in compilation database" % filename)
        return res

    def get_args_filename(self, filename, search_source_args=False):
        ret = []
        if self.global_args != None:
            ret = self.global_args

        ret += self.compile_args

        if self.cdb != None:
            cdb_ret = self.get_cdb_args(filename)
            if not cdb_ret and search_source_args:
                filename_base = os.path.splitext(filename)[0]
                for source_ext in SOURCE_EXTS:
                    cdb_ret = self.get_cdb_args(filename_base + source_ext)
                    if cdb_ret:
                        break
            ret += cdb_ret

        return ret

    def get_args_filename_ft(self, filename, filetype, search_source_args=False):
        ret = self.get_args_filename(filename, search_source_args)
        return ret + self.ft_compile_args[filetype]

    def get_available_args_files(self):
        return self.__paths

    @property
    def args_file(self):
        return self.__args_file
<|endoftext|>


<commit_before>#!/usr/bin/env python
# encoding: UTF-8

import argparse
from configparser import ConfigParser
import datetime
import logging
import platform
import sqlite3
import sys
import textwrap
import uuid

from cloudhands.burst.membership import handle_from_email
from cloudhands.burst.subscription import Online

from cloudhands.common.connectors import initialise
from cloudhands.common.connectors import Registry
from cloudhands.common.discovery import providers
from cloudhands.common.discovery import settings
from cloudhands.common.fsm import MembershipState
from cloudhands.common.fsm import RegistrationState
from cloudhands.common.fsm import SubscriptionState

from cloudhands.common.schema import Archive
from cloudhands.common.schema import CatalogueItem
from cloudhands.common.schema import Component
from cloudhands.common.schema import Directory
from cloudhands.common.schema import EmailAddress
from cloudhands.common.schema import Membership
from cloudhands.common.schema import Organisation
from cloudhands.common.schema import PosixUId
from cloudhands.common.schema import Provider
from cloudhands.common.schema import Registration
from cloudhands.common.schema import Subscription
from cloudhands.common.schema import Touch
from cloudhands.common.schema import User

from cloudhands.identity.registration import NewPassword

import cloudhands.web.indexer
import cloudhands.web.main
import cloudhands.common.schema

__doc__ = """
This program creates a fictional scenario for the purpose
of demonstrating the JASMIN web portal.
"""

DFLT_DB = ":memory:"

class WebFixture(object):

    subscriptions = [
        ("EOSCloud", "cloudhands.jasmin.vcloud.phase03.cfg"),
        ("NERC-EW", "cloudhands.jasmin.vcloud.phase03.cfg"),
        #("MARMITE", "cloudhands.jasmin.vcloud.phase02.cfg"),  # FIXME: SSL cert
    ]

    @staticmethod
    def demo_email(req=None):
        return "ben.campbell@universityoflife.ac.uk"

    @staticmethod
    def demo_password(req=None):
        return "IWannaS33TheDemo!"

    @staticmethod
    def create_subscriptions(session):
        log = logging.getLogger("cloudhands.web.demo.subscriptions")
        maintenance = session.query(
            SubscriptionState).filter(
            SubscriptionState.name=="maintenance").one()
        actor = session.query(Component).filter(
            Component.handle=="burst.controller").one()
        #if not actor:
        #    actor = Component(handle="Demo", uuid=uuid.uuid4().hex)
        #    session.add(actor)

        for orgName, providerName in WebFixture.subscriptions:
            org = session.query(Organisation).filter(
                Organisation.name==orgName).first()
            if not org:
                org = Organisation(
                    uuid=uuid.uuid4().hex,
                    name=orgName)
                session.add(org)

            provider = session.query(Provider).filter(
                Provider.name==providerName).first()
            if not provider:
                provider = Provider(
                    name=providerName, uuid=uuid.uuid4().hex)
                session.add(provider)

            if session.query(Subscription).join(Organisation).join(
                Provider).filter(Organisation.id==org.id).filter(
                Provider.id==provider.id).count():
                yield None
            else:

                subs = Subscription(
                    uuid=uuid.uuid4().hex,
                    model=cloudhands.common.__version__,
                    organisation=org,
                    provider=provider)
                subs.changes.append(
                    Touch(
                        artifact=subs, actor=actor, state=maintenance,
                        at=datetime.datetime.utcnow())
                    )
                act = Online(actor, subs)(session)
                subs.changes.append(act)
                try:
                    session.add(subs)
                    session.commit()
                    yield act
                except Exception as e:
                    log.debug(e)
                    session.rollback()
                finally:
                    session.flush()

    @staticmethod
    def create_catalogue(session):
        org = session.query(Organisation).first()
        try:
            session.add_all((
                CatalogueItem(
                    name="Bastion-host",
                    description="Headless VM for admin tasks",
                    note=textwrap.dedent("""

This VM runs OpenSSH on CentOS 6.5.
                        It is used for administration and monitoring tasks.

                        """),
                    logo="headless",
                    organisation=org,
                    uuid=uuid.uuid4().hex,
                ),
                CatalogueItem(
                    name="nfs-client",
                    description="Headless VM for file transfer operations",
                    note=textwrap.dedent("""

This VM runs CentOS 6.5 with a minimal amount of RAM and
                        no X server. It is used for file transfer operations from the
                        command line.

                        """),
                    logo="headless",
                    organisation=org,
                    uuid=uuid.uuid4().hex,
                ),
                CatalogueItem(
                    name="NFS-client-server",
                    description="Headless VM for file transfer operations",
                    note=textwrap.dedent("""

This VM runs NFS services on CentOS 6.5.

                        """),
                    logo="headless",
                    organisation=org,
                    uuid=uuid.uuid4().hex,
                ),
                CatalogueItem(
                    name="Web-Server",
                    description="Headless VM with Web server",
                    note=textwrap.dedent("""

This VM runs Apache on CentOS 6.5.
                        It has 8GB RAM and 4 CPU cores.
                        It is used for hosting websites and applications with a
                        Web API.

                        """),
                    logo="headless",
                    organisation=org,
                    uuid=uuid.uuid4().hex,
                ),
                CatalogueItem(
                    name="steve-pasco",
                    description="JASMIN Analysis Platform",
                    note=textwrap.dedent("""

This VM runs IPython notebook.
                        It has a wide range of uses for research and analysis.

                        """),
                    logo="headless",
                    organisation=org,
                    uuid=uuid.uuid4().hex,
                )
            ))
            session.commit()
        except Exception as e:
            session.rollback()
        finally:
            session.flush()

    @staticmethod
    def create_registration(session, user):
        preconfirm = session.query(RegistrationState).filter(
            RegistrationState.name == "pre_registration_inetorgperson").one()
        reg = Registration(
            uuid=uuid.uuid4().hex,
            model=cloudhands.common.__version__)
        act = NewPassword(user, WebFixture.demo_password(), reg)(session)
        ea = EmailAddress(touch=act, value=WebFixture.demo_email())
        try:
            session.add(ea)
            session.commit()
        except Exception as e:
            session.rollback()
        finally:
            session.flush()

        yield act

        confirmed = session.query(RegistrationState).filter(
            RegistrationState.name == "pre_user_posixaccount").one()
        now = datetime.datetime.utcnow()
        act = Touch(artifact=reg, actor=user, state=confirmed, at=now)

        # TODO: Formalise function (wildcard unpacking of forenames?)
        cn = ''.join(i[:n] for i, n in zip(
            reversed(user.handle.split()), (6, 1, 1))).lower()
        uid = PosixUId(value=cn, touch=act)
        # TODO: PosixUIdNumber of 7010001

        try:
            session.add(uid)
            session.commit()
        except Exception as e:
            session.rollback()
        finally:
            session.flush()

        yield act

    @staticmethod
    def grant_admin_memberships(session, user):
        log = logging.getLogger("cloudhands.web.demo.memberships")
        active = session.query(MembershipState).filter(
            MembershipState.name == "active").one()

        for name, provider in WebFixture.subscriptions:
            org = session.query(Organisation).filter(
                Organisation.name == name).one()

            ea = session.query(EmailAddress).filter(
                EmailAddress.value==WebFixture.demo_email()).first()
            if not ea:
                ea = EmailAddress(
                    value=WebFixture.demo_email())
                session.add(ea)

            if session.query(Membership).join(Organisation).join(
                Touch).join(User).filter(Organisation.id==org.id).filter(
                Membership.role=="admin").filter(User.id==user.id).count():
                yield None

            else:

                now = datetime.datetime.utcnow()
                mship = Membership(
                    uuid=uuid.uuid4().hex,
                    model=cloudhands.common.__version__,
                    organisation=org,
                    role="admin")
                act = Touch(artifact=mship, actor=user, state=active, at=now)
                ea.touch = act
                mship.changes.append(act)
                try:
                    session.add(ea)
                    session.commit()
                    yield act
                except Exception as e:
                    log.debug(e)
                    session.rollback()
                finally:
                    session.flush()

    @staticmethod
    def add_subscribed_resources(session, user):
        log = logging.getLogger("cloudhands.web.demo.resources")
        for name, provider in WebFixture.subscriptions:
            org = session.query(Organisation).filter(
                Organisation.name == name).one()
            grant = session.query(Touch).join(Membership).join(User).filter(
                Membership.organisation_id == org.id).filter(
                User.id == user.id).first()
            mship = grant.artifact
            for subs in org.subscriptions:
                now = datetime.datetime.utcnow()
                if isinstance(subs.provider, Archive):
                    d = Directory(
                        provider=subs.provider,
                        description="CIDER data archive",
                        mount_path="/{mount}/panfs/cider")
                    act = Touch(
                        artifact=mship, actor=user, state=grant.state, at=now)
                    d.touch = act
                    mship.changes.append(act)
                    try:
                        session.add_all((d, act))
                        session.commit()
                        yield act
                    except Exception as e:
                        session.rollback()
                    finally:
                        session.flush()

def main(args):
    logging.basicConfig(
        level=args.log_level,
        format="%(asctime)s %(levelname)-7s %(name)s|%(message)s")
    log = logging.getLogger("cloudhands.web.demo")

    args.interval = None
    args.query = None
    log.info("Generating index at {}".format(args.index))
    cloudhands.web.indexer.main(args)

    cfg, session = cloudhands.web.main.configure(args)

    for t in WebFixture.create_subscriptions(session):
        try:
            log.info("{} subscribed to {}".format(
                t.artifact.organisation.name,
                t.artifact.provider.name))
        except AttributeError:
            log.debug("Subscription pre-exists")

    WebFixture.create_catalogue(session)
    user = User(
        handle=handle_from_email(WebFixture.demo_email()),
        uuid=uuid.uuid4().hex)
    try:
        session.add(user)
        session.commit()
    except Exception as e:
        session.rollback()
    finally:
        session.flush()

    for t in WebFixture.create_registration(session, user):
        for r in t.resources:
            log.info(r)

    user = session.query(User).filter(User.handle == user.handle).one()
    for t in WebFixture.grant_admin_memberships(session, user):
        try:
            log.info("{} activated as admin for {}".format(
                t.actor.handle,
                t.artifact.organisation.name))
        except AttributeError:
            log.debug("Membership pre-exists for {}".format(user.handle))

    user = session.query(User).filter(User.handle == user.handle).one()
    for t in WebFixture.add_subscribed_resources(session, user):
        for r in t.resources:
            log.info("{} permitted access to a {} resource ({}).".format(
                t.actor.handle,
                r.provider.name,
                getattr(r, "description", "?")))

    cloudhands.web.main.authenticated_userid = WebFixture.demo_email

    app = cloudhands.web.main.wsgi_app(args, cfg)
    cloudhands.web.main.serve(
        app, host=platform.node(), port=args.port, url_scheme="http")
    return 0

def parser(description=__doc__):
    rv = cloudhands.web.main.parser(description)
    return rv

def run():
    p = parser()
    args = p.parse_args()
    rv = main(args)
    sys.exit(rv)

if __name__ == "__main__":
    run()
<commit_msg>Fixed integrity error in multiple runs of demo<commit_after>#!/usr/bin/env python
# encoding: UTF-8

import argparse
from configparser import ConfigParser
import datetime
import logging
import platform
import sqlite3
import sys
import textwrap
import uuid

from cloudhands.burst.membership import handle_from_email
from cloudhands.burst.subscription import Online

from cloudhands.common.connectors import initialise
from cloudhands.common.connectors import Registry
from cloudhands.common.discovery import providers
from cloudhands.common.discovery import settings
from cloudhands.common.fsm import MembershipState
from cloudhands.common.fsm import RegistrationState
from cloudhands.common.fsm import SubscriptionState

from cloudhands.common.schema import Archive
from cloudhands.common.schema import CatalogueItem
from cloudhands.common.schema import Component
from cloudhands.common.schema import Directory
from cloudhands.common.schema import EmailAddress
from cloudhands.common.schema import Membership
from cloudhands.common.schema import Organisation
from cloudhands.common.schema import PosixUId
from cloudhands.common.schema import Provider
from cloudhands.common.schema import Registration
from cloudhands.common.schema import Subscription
from cloudhands.common.schema import Touch
from cloudhands.common.schema import User

from cloudhands.identity.registration import NewPassword

import cloudhands.web.indexer
import cloudhands.web.main
import cloudhands.common.schema

__doc__ = """
This program creates a fictional scenario for the purpose
of demonstrating the JASMIN web portal.
"""

DFLT_DB = ":memory:"

class WebFixture(object):

    subscriptions = [
        ("EOSCloud", "cloudhands.jasmin.vcloud.phase03.cfg"),
        ("NERC-EW", "cloudhands.jasmin.vcloud.phase03.cfg"),
        #("MARMITE", "cloudhands.jasmin.vcloud.phase02.cfg"),  # FIXME: SSL cert
    ]

    @staticmethod
    def demo_email(req=None):
        return "ben.campbell@universityoflife.ac.uk"

    @staticmethod
    def demo_password(req=None):
        return "IWannaS33TheDemo!"

    @staticmethod
    def create_subscriptions(session):
        log = logging.getLogger("cloudhands.web.demo.subscriptions")
        maintenance = session.query(
            SubscriptionState).filter(
            SubscriptionState.name=="maintenance").one()
        actor = session.query(Component).filter(
            Component.handle=="burst.controller").one()
        #if not actor:
        #    actor = Component(handle="Demo", uuid=uuid.uuid4().hex)
        #    session.add(actor)

        for orgName, providerName in WebFixture.subscriptions:
            org = session.query(Organisation).filter(
                Organisation.name==orgName).first()
            if not org:
                org = Organisation(
                    uuid=uuid.uuid4().hex,
                    name=orgName)
                session.add(org)

            provider = session.query(Provider).filter(
                Provider.name==providerName).first()
            if not provider:
                provider = Provider(
                    name=providerName, uuid=uuid.uuid4().hex)
                session.add(provider)

            if session.query(Subscription).join(Organisation).join(
                Provider).filter(Organisation.id==org.id).filter(
                Provider.id==provider.id).count():
                yield None
            else:

                subs = Subscription(
                    uuid=uuid.uuid4().hex,
                    model=cloudhands.common.__version__,
                    organisation=org,
                    provider=provider)
                subs.changes.append(
                    Touch(
                        artifact=subs, actor=actor, state=maintenance,
                        at=datetime.datetime.utcnow())
                    )
                act = Online(actor, subs)(session)
                subs.changes.append(act)
                try:
                    session.add(subs)
                    session.commit()
                    yield act
                except Exception as e:
                    log.debug(e)
                    session.rollback()
                finally:
                    session.flush()

    @staticmethod
    def create_catalogue(session):
        org = session.query(Organisation).first()
        try:
            session.add_all((
                CatalogueItem(
                    name="Bastion-host",
                    description="Headless VM for admin tasks",
                    note=textwrap.dedent("""

This VM runs OpenSSH on CentOS 6.5.
                        It is used for administration and monitoring tasks.

                        """),
                    logo="headless",
                    organisation=org,
                    uuid=uuid.uuid4().hex,
                ),
                CatalogueItem(
                    name="nfs-client",
                    description="Headless VM for file transfer operations",
                    note=textwrap.dedent("""

This VM runs CentOS 6.5 with a minimal amount of RAM and
                        no X server. It is used for file transfer operations from the
                        command line.

                        """),
                    logo="headless",
                    organisation=org,
                    uuid=uuid.uuid4().hex,
                ),
                CatalogueItem(
                    name="NFS-client-server",
                    description="Headless VM for file transfer operations",
                    note=textwrap.dedent("""

This VM runs NFS services on CentOS 6.5.

                        """),
                    logo="headless",
                    organisation=org,
                    uuid=uuid.uuid4().hex,
                ),
                CatalogueItem(
                    name="Web-Server",
                    description="Headless VM with Web server",
                    note=textwrap.dedent("""

This VM runs Apache on CentOS 6.5.
                        It has 8GB RAM and 4 CPU cores.
                        It is used for hosting websites and applications with a
                        Web API.

                        """),
                    logo="headless",
                    organisation=org,
                    uuid=uuid.uuid4().hex,
                ),
                CatalogueItem(
                    name="steve-pasco",
                    description="JASMIN Analysis Platform",
                    note=textwrap.dedent("""

This VM runs IPython notebook.
                        It has a wide range of uses for research and analysis.

                        """),
                    logo="headless",
                    organisation=org,
                    uuid=uuid.uuid4().hex,
                )
            ))
            session.commit()
        except Exception as e:
            session.rollback()
        finally:
            session.flush()

    @staticmethod
    def create_registration(session, user):
        preconfirm = session.query(RegistrationState).filter(
            RegistrationState.name == "pre_registration_inetorgperson").one()
        reg = Registration(
            uuid=uuid.uuid4().hex,
            model=cloudhands.common.__version__)
        try:
            act = NewPassword(user, WebFixture.demo_password(), reg)(session)
        except Exception:
            session.rollback()
            session.flush()
            return

        ea = EmailAddress(touch=act, value=WebFixture.demo_email())
        try:
            session.add(ea)
            session.commit()
        except Exception as e:
            session.rollback()
        finally:
            session.flush()

        yield act

        confirmed = session.query(RegistrationState).filter(
            RegistrationState.name == "pre_user_posixaccount").one()
        now = datetime.datetime.utcnow()
        act = Touch(artifact=reg, actor=user, state=confirmed, at=now)

        # TODO: Formalise function (wildcard unpacking of forenames?)
        cn = ''.join(i[:n] for i, n in zip(
            reversed(user.handle.split()), (6, 1, 1))).lower()
        uid = PosixUId(value=cn, touch=act)
        # TODO: PosixUIdNumber of 7010001

        try:
            session.add(uid)
            session.commit()
        except Exception as e:
            session.rollback()
        finally:
            session.flush()

        yield act

    @staticmethod
    def grant_admin_memberships(session, user):
        log = logging.getLogger("cloudhands.web.demo.memberships")
        active = session.query(MembershipState).filter(
            MembershipState.name == "active").one()

        for name, provider in WebFixture.subscriptions:
            org = session.query(Organisation).filter(
                Organisation.name == name).one()

            ea = session.query(EmailAddress).filter(
                EmailAddress.value==WebFixture.demo_email()).first()
            if not ea:
                ea = EmailAddress(
                    value=WebFixture.demo_email())
                session.add(ea)

            if session.query(Membership).join(Organisation).join(
                Touch).join(User).filter(Organisation.id==org.id).filter(
                Membership.role=="admin").filter(User.id==user.id).count():
                yield None

            else:

                now = datetime.datetime.utcnow()
                mship = Membership(
                    uuid=uuid.uuid4().hex,
                    model=cloudhands.common.__version__,
                    organisation=org,
                    role="admin")
                act = Touch(artifact=mship, actor=user, state=active, at=now)
                ea.touch = act
                mship.changes.append(act)
                try:
                    session.add(ea)
                    session.commit()
                    yield act
                except Exception as e:
                    log.debug(e)
                    session.rollback()
                finally:
                    session.flush()

    @staticmethod
    def add_subscribed_resources(session, user):
        log = logging.getLogger("cloudhands.web.demo.resources")
        for name, provider in WebFixture.subscriptions:
            org = session.query(Organisation).filter(
                Organisation.name == name).one()
            grant = session.query(Touch).join(Membership).join(User).filter(
                Membership.organisation_id == org.id).filter(
                User.id == user.id).first()
            mship = grant.artifact
            for subs in org.subscriptions:
                now = datetime.datetime.utcnow()
                if isinstance(subs.provider, Archive):
                    d = Directory(
                        provider=subs.provider,
                        description="CIDER data archive",
                        mount_path="/{mount}/panfs/cider")
                    act = Touch(
                        artifact=mship, actor=user, state=grant.state, at=now)
                    d.touch = act
                    mship.changes.append(act)
                    try:
                        session.add_all((d, act))
                        session.commit()
                        yield act
                    except Exception as e:
                        session.rollback()
                    finally:
                        session.flush()

def main(args):
    logging.basicConfig(
        level=args.log_level,
        format="%(asctime)s %(levelname)-7s %(name)s|%(message)s")
    log = logging.getLogger("cloudhands.web.demo")

    args.interval = None
    args.query = None
    log.info("Generating index at {}".format(args.index))
    cloudhands.web.indexer.main(args)

    cfg, session = cloudhands.web.main.configure(args)

    for t in WebFixture.create_subscriptions(session):
        try:
            log.info("{} subscribed to {}".format(
                t.artifact.organisation.name,
                t.artifact.provider.name))
        except AttributeError:
            log.debug("Subscription pre-exists")

    WebFixture.create_catalogue(session)
    user = User(
        handle=handle_from_email(WebFixture.demo_email()),
        uuid=uuid.uuid4().hex)
    try:
        session.add(user)
        session.commit()
    except Exception as e:
        session.rollback()
    finally:
        session.flush()

    for t in WebFixture.create_registration(session, user):
        for r in t.resources:
            log.info(r)

    user = session.query(User).filter(User.handle == user.handle).one()
    for t in WebFixture.grant_admin_memberships(session, user):
        try:
            log.info("{} activated as admin for {}".format(
                t.actor.handle,
                t.artifact.organisation.name))
        except AttributeError:
            log.debug("Membership pre-exists for {}".format(user.handle))

    user = session.query(User).filter(User.handle == user.handle).one()
    for t in WebFixture.add_subscribed_resources(session, user):
        for r in t.resources:
            log.info("{} permitted access to a {} resource ({}).".format(
                t.actor.handle,
                r.provider.name,
                getattr(r, "description", "?")))

    cloudhands.web.main.authenticated_userid = WebFixture.demo_email

    app = cloudhands.web.main.wsgi_app(args, cfg)
    cloudhands.web.main.serve(
        app, host=platform.node(), port=args.port, url_scheme="http")
    return 0

def parser(description=__doc__):
    rv = cloudhands.web.main.parser(description)
    return rv

def run():
    p = parser()
    args = p.parse_args()
    rv = main(args)
    sys.exit(rv)

if __name__ == "__main__":
    run()
<|endoftext|>


<commit_before>"""
The game_state entities module contains the classes that represent
game entities for suspects, weapons, rooms, hallways, game cards,
and the game board
"""

#constants for suspect names
SUSPECT = 'Suspect'

MUSTARD = 'Colonel Mustard'
SCARLET = 'Miss Scarlet'
PLUM = 'Professor Plum'
GREEN = 'Mr. Green'
WHITE = 'Mrs. White'
PEACOCK = 'Mrs. Peacock'

SUSPECTS = [
    MUSTARD,
    SCARLET,
    PLUM,
    GREEN,
    WHITE,
    PEACOCK
]

#constants for weapon names
WEAPON = 'Weapon'

ROPE = 'Rope'
LEAD_PIPE = 'Lead Pipe'
KNIFE = 'Knife'
WRENCH = 'Wrench'
CANDLESTICK = 'Candlestick'
REVOLVER = 'Revolver'

WEAPONS = [
    ROPE,
    LEAD_PIPE,
    KNIFE,
    WRENCH,
    CANDLESTICK,
    REVOLVER
]

#constants for rooms
ROOM = 'Room'
KITCHEN = 'Kitchen'
BALLROOM = 'Ballroom'
CONSERVATORY = 'Conservatory'
BILLIARD_ROOM = 'Billiard Room'
LIBRARY = 'Library'
STUDY = 'Study'
HALL = 'Hall'
LOUNGE = 'Lounge'
DINING_ROOM = 'Dining Room'

ROOMS = [
    KITCHEN,
    BALLROOM,
    CONSERVATORY,
    BILLIARD_ROOM,
    LIBRARY,
    STUDY,
    HALL,
    LOUNGE,
    DINING_ROOM
]

#constants for hallways
STUDY_LIBRARY = 'study-library hallway'
STUDY_HALL = 'study-hall ahllway'
HALL_BILLIARD = 'hall-billiard hallway'
HALL_LOUNGE = 'hall-lounge hallway'
LOUNGE_DINING = 'lounge-dining hallway'
LIBRARY_CONSERVATORY = 'library-conservatory hallway'
LIBRARY_BILLIARD = 'library-billiard hallway'
BILLIARD_BALLROOM = 'billiard-ballroom hallway'
BILLIARD_DINING = 'billiard-dining hallway'
DINING_KITCHEN = 'dining-kitchen hallway'
CONSERVATORY_BALLROOM = 'conservatory-ballroom hallway'
BALLROOM_KITCHEN = "ballroom-kitchen hallway"

HALLWAY = [
    STUDY_LIBRARY,
    STUDY_HALL,
    HALL_BILLIARD,
    HALL_LOUNGE,
    LOUNGE_DINING,
    LIBRARY_CONSERVATORY,
    LIBRARY_BILLIARD,
    BILLIARD_BALLROOM,
    BILLIARD_DINING,
    DINING_KITCHEN,
    CONSERVATORY_BALLROOM,
    BALLROOM_KITCHEN
]

#constants for turn state
AWAITING_MOVE = "Waiting for player to move"
AWAITING_SUGGESTION = "Waiting for player to make suggestion"
AWAITING_SUGGESTION_RESPONSE = "Waiting for response to player suggestion"

class Player(object):
    """
    Player is an external user fo the system that will be playing the game
    """
    def __init__(self, username):
        self.username = username
        self.suspect = None
        self.game_cards = list()

class GameSpace(object):
    """
    GameSpace is an abstract super class used to define
    spaces on the game board
    """
    def __init__(self, name, connected_spaces, suspects=None):
        self.name = name
        self.connected_spaces = connected_spaces
        if suspects:
            self.suspects = suspects
        else:
            self.suspects = list()

    def is_available(self):
        raise NotImplementedError

class Room(GameSpace):
    """
    Room represents a room on the game board.
    It may contain suspects and weapons
    """
    def __init__(self, name, connected_spaces):
        super(Room, self).__init__(name, connected_spaces)
        self.suspects = None
        self.weapons = None

    def is_available(self):
        #A Room can hold more than one suspect at a time
        #so it will always be available
        return True

class Hallway(GameSpace):
    """
    Hallways represents the hallways between rooms on the GameBoard
    """
    def __init__(self, name, connected_spaces):
        super(Hallway, self).__init__(name, connected_spaces)
        self.suspect = None

    def is_available(self):
        #Only one suspect can be in a hallway at a time, so return
        #True if empty or False if occupied
        return not self.suspect

class HomeSquare(GameSpace):
    """
    HomeSquare represent the starting square for each suspect on the
    game board.  Starting squares may only have one occupant at a time
    and can not be moved to during a turn
    """
    def __init__(self, name, connected_spaces, suspects=None):
        super(HomeSquare, self).__init__(name, connected_spaces, suspects=None)
        self.suspect = None

    def is_available(self):
        #returns false because players can not move to
        #HomeSquares during regular play
        return False

class GameCard(object):
    """
    represents a single Clue-less game card
    """
    def __init__(self, item, item_type):
        self.item = item
        self.type = item_type

class GameState(object):
    """
    GameState  creates all properties necessary to
    observe the state fo the Game in play
    """
    def __init__(self, players):
        self.players = players
        self.player_messages = list()

        self.turn_list = list(players)
        self.current_player = self.turn_list[0]
        self.turn_status = AWAITING_MOVE

        #holds the winning cards
        self.case_file = None

        #The game_board is a collections of HomeSquares, Rooms, and Hallways,
        #and defines their relation to one another on the game board
        self.game_board = {
            KITCHEN: Room(
                name=KITCHEN,
                connected_spaces=[
                    DINING_KITCHEN,
                    BALLROOM_KITCHEN,
                    STUDY
                ]
            ),
            BALLROOM: Room(
                name=BALLROOM,
                connected_spaces=[
                    BALLROOM_KITCHEN,
                    CONSERVATORY_BALLROOM,
                    BILLIARD_BALLROOM
                ]
            ),
            CONSERVATORY: Room(
                name=CONSERVATORY,
                connected_spaces=[
                    CONSERVATORY_BALLROOM,
                    LIBRARY_CONSERVATORY,
                    LOUNGE
                ]
            ),
            BILLIARD_ROOM: Room(
                name=BILLIARD_ROOM,
                connected_spaces=[
                    HALL_BILLIARD,
                    LIBRARY_BILLIARD,
                    BILLIARD_BALLROOM,
                    BILLIARD_DINING
                ]
            ),
            LIBRARY: Room(
                name=LIBRARY,
                connected_spaces=[
                    STUDY_LIBRARY,
                    LIBRARY_CONSERVATORY,
                    LIBRARY_BILLIARD
                ]
            ),
            STUDY: Room(
                name=STUDY,
                connected_spaces=[
                    KITCHEN,
                    STUDY_LIBRARY,
                    STUDY_HALL
                ]
            ),
            HALL: Room(
                name=HALL,
                connected_spaces=[
                    STUDY_HALL,
                    HALL_BILLIARD,
                    HALL_LOUNGE
                ]
            ),
            LOUNGE: Room(
                name=LOUNGE,
                connected_spaces=[
                    CONSERVATORY,
                    HALL_LOUNGE,
                    LOUNGE_DINING
                ]
            ),
            DINING_ROOM: Room(
                name=DINING_ROOM,
                connected_spaces=[
                    LOUNGE_DINING,
                    BILLIARD_DINING,
                    DINING_KITCHEN
                ]
            ),
            STUDY_LIBRARY: Hallway(
                name=STUDY_LIBRARY,
                connected_spaces=[
                    STUDY,
                    LIBRARY
                ]
            ),
            STUDY_HALL: Hallway(
                name=STUDY_HALL,
                connected_spaces=[
                    STUDY,
                    HALL
                ]
            ),
            HALL_BILLIARD: Hallway(
                name=HALL_BILLIARD,
                connected_spaces=[
                    HALL,
                    BILLIARD_ROOM
                ]
            ),
            HALL_LOUNGE: Hallway(
                name=HALL_LOUNGE,
                connected_spaces=[
                    HALL,
                    LOUNGE
                ]
            ),
            LOUNGE_DINING: Hallway(
                name=LOUNGE_DINING,
                connected_spaces=[
                    LOUNGE,
                    DINING_ROOM
                ]
            ),
            LIBRARY_CONSERVATORY: Hallway(
                name=LIBRARY_CONSERVATORY,
                connected_spaces=[
                    LIBRARY,
                    CONSERVATORY
                ]
            ),
            LIBRARY_BILLIARD: Hallway(
                name=LIBRARY_BILLIARD,
                connected_spaces=[
                    LIBRARY,
                    BILLIARD_ROOM
                ]
            ),
            BILLIARD_BALLROOM:  Hallway(
                name=BILLIARD_BALLROOM,
                connected_spaces=[
                    BILLIARD_ROOM,
                    BALLROOM
                ]
            ),
            BILLIARD_DINING: Hallway(
                name=BILLIARD_DINING,
                connected_spaces=[
                    BILLIARD_ROOM,
                    DINING_ROOM
                ]
            ),
            DINING_KITCHEN: Hallway(
                name=DINING_KITCHEN,
                connected_spaces=[
                    DINING_ROOM,
                    KITCHEN
                ]
            ),
            CONSERVATORY_BALLROOM: Hallway(
                name=CONSERVATORY_BALLROOM,
                connected_spaces=[
                    CONSERVATORY,
                    BALLROOM_KITCHEN
                ]
            ),
            BALLROOM_KITCHEN:  Hallway(
                name=BALLROOM_KITCHEN,
                connected_spaces=[
                    BALLROOM,
                    KITCHEN
                ]
            ),
            MUSTARD: HomeSquare(
                name=MUSTARD,
                connected_spaces=[
                    LOUNGE_DINING
                ],
                suspects=[MUSTARD]
            ),
            SCARLET: HomeSquare(
                name=SCARLET,
                connected_spaces=[
                    HALL_LOUNGE
                ],
                suspects=[SCARLET]
            ),
            PLUM: HomeSquare(
                name=PLUM,
                connected_spaces=[
                    STUDY_LIBRARY
                ],
                suspects=[PLUM]
            ),
            GREEN: HomeSquare(
                name=GREEN,
                connected_spaces=[
                    CONSERVATORY_BALLROOM
                ],
                suspects=[GREEN]
            ),
            WHITE: HomeSquare(
                name=WHITE,
                connected_spaces=[
                    BALLROOM_KITCHEN
                ],
                suspects=[WHITE]
            ),
            PEACOCK: HomeSquare(
                name=PEACOCK,
                connected_spaces=[
                    LIBRARY_CONSERVATORY
                ],
                suspects=[PEACOCK]
            )
        }
<commit_msg>add format() methods to game_state objects, and add optional args to constructors.  This will allow all objects to be recreated from their format methods<commit_after>"""
The game_state entities module contains the classes that represent
game entities for suspects, weapons, rooms, hallways, game cards,
and the game board
"""

#constants for suspect names
SUSPECT = 'Suspect'

MUSTARD = 'Colonel Mustard'
SCARLET = 'Miss Scarlet'
PLUM = 'Professor Plum'
GREEN = 'Mr. Green'
WHITE = 'Mrs. White'
PEACOCK = 'Mrs. Peacock'

SUSPECTS = [
    MUSTARD,
    SCARLET,
    PLUM,
    GREEN,
    WHITE,
    PEACOCK
]

#constants for weapon names
WEAPON = 'Weapon'

ROPE = 'Rope'
LEAD_PIPE = 'Lead Pipe'
KNIFE = 'Knife'
WRENCH = 'Wrench'
CANDLESTICK = 'Candlestick'
REVOLVER = 'Revolver'

WEAPONS = [
    ROPE,
    LEAD_PIPE,
    KNIFE,
    WRENCH,
    CANDLESTICK,
    REVOLVER
]

#constants for rooms
ROOM = 'Room'
KITCHEN = 'Kitchen'
BALLROOM = 'Ballroom'
CONSERVATORY = 'Conservatory'
BILLIARD_ROOM = 'Billiard Room'
LIBRARY = 'Library'
STUDY = 'Study'
HALL = 'Hall'
LOUNGE = 'Lounge'
DINING_ROOM = 'Dining Room'

ROOMS = [
    KITCHEN,
    BALLROOM,
    CONSERVATORY,
    BILLIARD_ROOM,
    LIBRARY,
    STUDY,
    HALL,
    LOUNGE,
    DINING_ROOM
]

#constants for hallways
STUDY_LIBRARY = 'study-library hallway'
STUDY_HALL = 'study-hall ahllway'
HALL_BILLIARD = 'hall-billiard hallway'
HALL_LOUNGE = 'hall-lounge hallway'
LOUNGE_DINING = 'lounge-dining hallway'
LIBRARY_CONSERVATORY = 'library-conservatory hallway'
LIBRARY_BILLIARD = 'library-billiard hallway'
BILLIARD_BALLROOM = 'billiard-ballroom hallway'
BILLIARD_DINING = 'billiard-dining hallway'
DINING_KITCHEN = 'dining-kitchen hallway'
CONSERVATORY_BALLROOM = 'conservatory-ballroom hallway'
BALLROOM_KITCHEN = "ballroom-kitchen hallway"

HALLWAY = [
    STUDY_LIBRARY,
    STUDY_HALL,
    HALL_BILLIARD,
    HALL_LOUNGE,
    LOUNGE_DINING,
    LIBRARY_CONSERVATORY,
    LIBRARY_BILLIARD,
    BILLIARD_BALLROOM,
    BILLIARD_DINING,
    DINING_KITCHEN,
    CONSERVATORY_BALLROOM,
    BALLROOM_KITCHEN
]

#constants for turn state
AWAITING_MOVE = "Waiting for player to move"
AWAITING_SUGGESTION = "Waiting for player to make suggestion"
AWAITING_SUGGESTION_RESPONSE = "Waiting for response to player suggestion"

class Player(object):
    """
    Player is an external user fo the system that will be playing the game
    """
    def __init__(self, username, suspect=None, game_cards=None):
        self.username = username
        self.suspect = None
        if game_cards:
            self.game_cards = game_cards
        else:
            self.game_cards = list()

    def format(self):
        """
        format the object as a dictionary.
        """
        return {
            "username": self.username,
            "suspect": self.suspect,
            "game_cards": [
                game_card.format() for game_card in self.game_cards
            ]
        }

class GameSpace(object):
    """
    GameSpace is an abstract super class used to define
    spaces on the game board
    """
    def __init__(self, name, connected_spaces, suspects=None):
        self.name = name
        self.connected_spaces = connected_spaces
        if suspects:
            self.suspects = suspects
        else:
            self.suspects = list()

    def is_available(self):
        raise NotImplementedError

    def format(self):
        """
        format the object as a dictionary.
        """
        return {
            "name": self.name,
            "connected_spaces":  self.connected_spaces,
            "suspects": self.suspects,
            "weapons": self.weapons
        }

class Room(GameSpace):
    """
    Room represents a room on the game board.
    It may contain suspects and weapons
    """
    def __init__(self, name, connected_spaces, suspects=None, weapons=None):
        super(Room, self).__init__(name, connected_spaces, suspects)

        if weapons:
            self.weapons = weapons
        else:
            self.weapons = None

    def is_available(self):
        #A Room can hold more than one suspect at a time
        #so it will always be available
        return True

    def format(self):
        """
        format the object as a dictionary.  This moethod overrides the
        superclass format method in order to add the weapons attribute
        """
        return {
            "name": self.name,
            "connected_spaces": [
                space.format() for space in self.connected_spaces
            ],
            "suspects": self.suspects,
            "weapons": self.weapons
        }

class Hallway(GameSpace):
    """
    Hallways represents the hallways between rooms on the GameBoard
    """
    def __init__(self, name, connected_spaces, suspects=None):
        super(Hallway, self).__init__(name, connected_spaces, suspects)

    def is_available(self):
        #Only one suspect can be in a hallway at a time, so return
        #True if empty or False if occupied
        return not self.suspects

class HomeSquare(GameSpace):
    """
    HomeSquare represent the starting square for each suspect on the
    game board.  Starting squares may only have one occupant at a time
    and can not be moved to during a turn
    """
    def __init__(self, name, connected_spaces, suspects=None):
        super(HomeSquare, self).__init__(name, connected_spaces, suspects)

    def is_available(self):
        #returns false because players can not move to
        #HomeSquares during regular play
        return False

class GameCard(object):
    """
    represents a single Clue-less game card
    """
    def __init__(self, item, item_type):
        self.item = item
        self.type = item_type

    def format(self):
        """
        format the object as a dictionary.
        """
        return {
            "item": self.item,
            "type": self.type
        }

class GameState(object):
    """
    GameState  creates all properties necessary to
    observe the state fo the Game in play
    """
    def __init__(self, players, player_messages=None, turn_list=None,
                 current_player=None, turn_status=None, case_file=None,
                 game_board=None):
        self.players = players
        if player_messages:
            self.player_messages = player_messages
        else:
            self.player_messages = list()
        if turn_list:
            self.turn_list = turn_list
        else:
            self.turn_list = list(players)
        if current_player:
            self.current_player = current_player
        else:
            self.current_player = self.turn_list[0]
        if turn_status:
            self.turn_status = turn_status
        else:
            self.turn_status = AWAITING_MOVE

        #holds the winning cards
        if case_file:
            self.case_file = case_file
        else:
            self.case_file = None

        #The game_board is a collections of HomeSquares, Rooms, and Hallways,
        #and defines their relation to one another on the game board
        if game_board:
            self.game_board = game_board
        else:
            self.game_board = {
                KITCHEN: Room(
                    name=KITCHEN,
                    connected_spaces=[
                        DINING_KITCHEN,
                        BALLROOM_KITCHEN,
                        STUDY
                    ]
                ),
                BALLROOM: Room(
                    name=BALLROOM,
                    connected_spaces=[
                        BALLROOM_KITCHEN,
                        CONSERVATORY_BALLROOM,
                        BILLIARD_BALLROOM
                    ]
                ),
                CONSERVATORY: Room(
                    name=CONSERVATORY,
                    connected_spaces=[
                        CONSERVATORY_BALLROOM,
                        LIBRARY_CONSERVATORY,
                        LOUNGE
                    ]
                ),
                BILLIARD_ROOM: Room(
                    name=BILLIARD_ROOM,
                    connected_spaces=[
                        HALL_BILLIARD,
                        LIBRARY_BILLIARD,
                        BILLIARD_BALLROOM,
                        BILLIARD_DINING
                    ]
                ),
                LIBRARY: Room(
                    name=LIBRARY,
                    connected_spaces=[
                        STUDY_LIBRARY,
                        LIBRARY_CONSERVATORY,
                        LIBRARY_BILLIARD
                    ]
                ),
                STUDY: Room(
                    name=STUDY,
                    connected_spaces=[
                        KITCHEN,
                        STUDY_LIBRARY,
                        STUDY_HALL
                    ]
                ),
                HALL: Room(
                    name=HALL,
                    connected_spaces=[
                        STUDY_HALL,
                        HALL_BILLIARD,
                        HALL_LOUNGE
                    ]
                ),
                LOUNGE: Room(
                    name=LOUNGE,
                    connected_spaces=[
                        CONSERVATORY,
                        HALL_LOUNGE,
                        LOUNGE_DINING
                    ]
                ),
                DINING_ROOM: Room(
                    name=DINING_ROOM,
                    connected_spaces=[
                        LOUNGE_DINING,
                        BILLIARD_DINING,
                        DINING_KITCHEN
                    ]
                ),
                STUDY_LIBRARY: Hallway(
                    name=STUDY_LIBRARY,
                    connected_spaces=[
                        STUDY,
                        LIBRARY
                    ]
                ),
                STUDY_HALL: Hallway(
                    name=STUDY_HALL,
                    connected_spaces=[
                        STUDY,
                        HALL
                    ]
                ),
                HALL_BILLIARD: Hallway(
                    name=HALL_BILLIARD,
                    connected_spaces=[
                        HALL,
                        BILLIARD_ROOM
                    ]
                ),
                HALL_LOUNGE: Hallway(
                    name=HALL_LOUNGE,
                    connected_spaces=[
                        HALL,
                        LOUNGE
                    ]
                ),
                LOUNGE_DINING: Hallway(
                    name=LOUNGE_DINING,
                    connected_spaces=[
                        LOUNGE,
                        DINING_ROOM
                    ]
                ),
                LIBRARY_CONSERVATORY: Hallway(
                    name=LIBRARY_CONSERVATORY,
                    connected_spaces=[
                        LIBRARY,
                        CONSERVATORY
                    ]
                ),
                LIBRARY_BILLIARD: Hallway(
                    name=LIBRARY_BILLIARD,
                    connected_spaces=[
                        LIBRARY,
                        BILLIARD_ROOM
                    ]
                ),
                BILLIARD_BALLROOM:  Hallway(
                    name=BILLIARD_BALLROOM,
                    connected_spaces=[
                        BILLIARD_ROOM,
                        BALLROOM
                    ]
                ),
                BILLIARD_DINING: Hallway(
                    name=BILLIARD_DINING,
                    connected_spaces=[
                        BILLIARD_ROOM,
                        DINING_ROOM
                    ]
                ),
                DINING_KITCHEN: Hallway(
                    name=DINING_KITCHEN,
                    connected_spaces=[
                        DINING_ROOM,
                        KITCHEN
                    ]
                ),
                CONSERVATORY_BALLROOM: Hallway(
                    name=CONSERVATORY_BALLROOM,
                    connected_spaces=[
                        CONSERVATORY,
                        BALLROOM_KITCHEN
                    ]
                ),
                BALLROOM_KITCHEN:  Hallway(
                    name=BALLROOM_KITCHEN,
                    connected_spaces=[
                        BALLROOM,
                        KITCHEN
                    ]
                ),
                MUSTARD: HomeSquare(
                    name=MUSTARD,
                    connected_spaces=[
                        LOUNGE_DINING
                    ],
                    suspects=[MUSTARD]
                ),
                SCARLET: HomeSquare(
                    name=SCARLET,
                    connected_spaces=[
                        HALL_LOUNGE
                    ],
                    suspects=[SCARLET]
                ),
                PLUM: HomeSquare(
                    name=PLUM,
                    connected_spaces=[
                        STUDY_LIBRARY
                    ],
                    suspects=[PLUM]
                ),
                GREEN: HomeSquare(
                    name=GREEN,
                    connected_spaces=[
                        CONSERVATORY_BALLROOM
                    ],
                    suspects=[GREEN]
                ),
                WHITE: HomeSquare(
                    name=WHITE,
                    connected_spaces=[
                        BALLROOM_KITCHEN
                    ],
                    suspects=[WHITE]
                ),
                PEACOCK: HomeSquare(
                    name=PEACOCK,
                    connected_spaces=[
                        LIBRARY_CONSERVATORY
                    ],
                    suspects=[PEACOCK]
                )
            }

    def format(self):
        """
        format the object as a dictionary.
        """
        return {
            "players": [
                player.format() for player in self.players
            ],
            "player_messages": self.player_messages,
            "turn_list": [
                player.format() for player in self.turn_list
            ],
            "current_player": self.current_player.format(),
            "turn_status": self.turn_status,
            "case_file": [card.format() for card in self.case_file],
            "game_board": [
                game_space.format() for game_space in self.game_board
            ]
        }
<|endoftext|>


<commit_before>import logging
log = logging.getLogger(__name__)

import re
import time

# For determining hostname
import socket

# Events, for Task
import events

# Subprocess, for NagiosTask
import shlex
import subprocess

# Multiprocessing, for PythonTask
import multiprocessing

# Requests, for CloudKickTask
import requests

# JSON, for JSONTask
import json

# Some module-level constants for definining tasks
DEFAULT_TTL = 60
DEFAULT_HOSTNAME = socket.gethostname()
DEFAULT_MULTIPLIER = 5

SEED_TIMING = 0.5

# Numeric matcher for parsing Nagios performance data
# create class constant compiled regex, to prevent
# unecessarily recompiling this regex string.
NUMERIC_REGEX = re.compile('([\d.]+)')

# Configure logging for the Requests module, decreasing
# the verbosity even when we are running in a more
# verbose mode otherwise.
requests_log = logging.getLogger("requests")
requests_log.setLevel(logging.WARNING)

class Task():
    required_config = ['service', 'arg']

    def __init__(self, config):
        self.config = config

        # Confirm that the passed config object contains required values
        for key in self.required_config:
            if key not in config:
                raise KeyError("Config missing '%s'" % (key))

        self.name = config['service']
        self.arg = config['arg']

        # Handle some common task options and their defaults
        self.ttl_multiplier = config['ttl_multiplier'] if 'ttl_multiplier' in config else DEFAULT_MULTIPLIER
        self.host = config['host'] if 'host' in config else DEFAULT_HOSTNAME
        self.ttl = config['ttl'] if 'ttl' in config else DEFAULT_TTL

        self.attributes = config['attributes'] if 'attributes' in config else {}
        self.tags = set(config['tags']) if 'tags' in config else set()
        self.note = config['note'] if 'note' in config else ""

        # Initialize task internal values
        self.events = []
        self.timings = [SEED_TIMING]
        self.locked = False

        log.info("Creating task: '%s' with TTL of %ss" % (self.name, self.ttl))

    def add_timing(self, value, keep=5):
        log.debug("Task %s took %0.2fs" % (self.name, value))
        self.timings.append(value)
        del self.timings[:-keep]

    def skew(self):
        return sum(self.timings)/len(self.timings)

    def start(self):
        if not self.locked:
            log.info("Starting task: '%s' (usually takes %0.2fs)" % (self.name, self.skew()))
            self.locked = True
            self.start_time = time.time()
            self.run()
        else:
            raise RuntimeError("Task '%s' is locked - cannot start another." % (self.name))

    def drain(self):
        self.join()
        self.end_time = time.time()
        self.add_timing(self.end_time - self.start_time)

        tevents = []
        while len(self.events) > 0:
            tevents.append(self.events.pop(0))

        # Now that we've got our events, free the task to run again
        self.locked = False
        return tevents

class HTTPJSONTask(Task):
    def __init__(self, config):
        Task.__init__(self, config)

    def request(self, q):
        try:
            log.debug("Starting web request to '%s'" % (self.arg))
            resp = requests.get(self.arg)
            q.put(resp.json(), timeout=(self.ttl * 0.3))
        except Exception as e:
            log.error("Exception during request method of CloudKickTask '%s'\n%s" % (self.name, str(e)))

    def run(self):
        try:
            self.q = multiprocessing.Queue()
            self.proc = multiprocessing.Process(target=self.request, args=(self.q))
            self.proc.start()
        except Exception as e:
            log.error("Exception starting CloudKickTask '%s'\n%s" % (self.name, str(e)))

    def join(self):
        try:
            json_result = self.q.get(timeout=(self.ttl * 0.3))
            self.proc.join()

            log.debug('CloudKickTask: Processing %s metrics' % (len(json_result['metrics'])))
            for metric in json_result['metrics']:
                note = metric['note'] if 'note' in metric else self.note
                event = events.Event()
                event.ttl = self.ttl * self.ttl_multiplier
                event.host = self.host
                event.tags = self.tags
                event.service = metric['name']
                event.state = metric['state']
                event.metric = metric['value']
                event.description = "%s\nWarn threshold: %s, Error threshold: %s" % (note,
                                                                                metric['warn_threshold'],
                                                                                metric['error_threshold'])
                self.events.append(event)
        except Exception as e:
            log.error("Exception joining CloudKickTask '%s'\n%s" % (self.name, str(e)))

class SubProcessTask(Task):
    def __init__(self, config):
        Task.__init__(self, config)

        self.raw_command = self.arg
        self.command = shlex.split(self.arg)
        self.process = None
        self.use_shell = False
        self.attrprefix = 'task_'

    def run(self):
        try:
            self.process = subprocess.Popen(self.command, stdout=subprocess.PIPE, shell=self.use_shell)
        except Exception as e:
            log.error("Exception running task '%s':\n%s" % (self.name, str(e)))

    def join(self):
        try:
            deadline = self.start_time + (self.ttl * 0.5)
            while deadline > time.time():
                if self.process.poll() == None:
                    time.sleep(0.5)
                else:
                    log.debug("Gathering output from task '%s'" % (self.name))
                    stdout, sterr = self.process.communicate()
                    return stdout, sterr, self.process.returncode
            else:
                log.warning("Deadline expired for task '%s' - force killing" % (self.name))
                self.process.kill()
                self.process.wait()
                log.debug("Subprocess killed for task '%s'" % (self.name))
                return '', '', -127
        except Exception as e:
            log.error("Exception joining task '%s':\n%s" % (self.name, str(e)))

class NagiosTask(SubProcessTask):
    exitcodes = {
     -127: 'timeout',
        0: 'ok',
        1: 'warn',
        2: 'critical',
        3: 'unknown'
    }

    def __init__(self, config):
        SubProcessTask.__init__(self, config)

    def parse_nagios_output(self, stdout):
        # TODO: Be more robust if this fails for any reason
        parts = stdout.split("|")
        if len(parts) == 1:
            log.debug("Task '%s' did not return perf data." % (self.name))
            return (parts[0], None)
        elif len(parts) == 2:
            attributes = {}
            output, raw_perf = parts
            log.debug("Task '%s' returned perf data: %s" % (self.name, raw_perf.strip()))
            for item in raw_perf.strip().split(" "):
                key, val = item.split(';')[0].split('=')
                attributes[self.attrprefix + key] = val

            return output, attributes
        else:
            log.warning("Output for task '%s' could not be parsed for perf data: %s" % (self.name, str(parts)))
            return (stdout, None)

    def join(self):
        try:
            stdout, stderr, returncode = SubProcessTask.join(self)

            event = events.Event()
            event.service = self.name
            event.ttl = self.ttl * self.ttl_multiplier
            event.host = self.host
            event.tags = self.tags
            event.attributes = self.attributes

            output, attributes = self.parse_nagios_output(stdout)
            event.description = "Note: %s\nCommand: %s\nOutput: %s\n" % (self.note, self.raw_command, output)

            if returncode in self.exitcodes:
                event.state = self.exitcodes[returncode]
            else:
                event.state = 'unknown'

            if attributes:
                if 'metric' in config and config['metric'] in attributes:
                    metric_key = config['metric']
                else:
                    metric_key = attributes.keys()[0]

                event.attributes.update(attributes)
                event.metric = float(NUMERIC_REGEX.match(attributes[metric_key]).group(1))

            self.events.append(event)
        except Exception as e:
            log.error("Exception joining task '%s':\n%s" % (self.name, str(e)))

class JSONTask(SubProcessTask):
    def __init__(self, config):
        SubProcessTask.__init__(self, config)

    def join(self):
        try:
            stdout, stderr, returncode = SubProcessTask.join(self)

            try:
                results = json.loads(stdout)
            except Exception as e:
                log.error("Failed to parse JSON. '%s':\n%s" % (self.name, str(e)))

            for result in results:
                for field in ['service', 'state', 'description', 'metric']:
                    if field not in result:
                        log.error("Event missing field '%s'" % (field))
                        continue

                event = events.Event()
                event.ttl = self.ttl * self.ttl_multiplier
                event.host = self.host
                event.tags = self.tags
                event.attributes = self.attributes

                if "attributes" in result:
                    event.attributes.update({self.attrprefix + name: result["attributes"][name] for name in result["attributes"]})

                event.service = result['service']
                event.state = result['state']
                event.metric = result['metric']
                event.description = "%s\n%s" % (self.note, result['description'])
                self.events.append(event)
        except Exception as e:
            log.error("Exception joining task '%s':\n%s" % (self.name, str(e)))
<commit_msg>Only instantiate queue once per class - we can reuse it<commit_after>import logging
log = logging.getLogger(__name__)

import re
import time

# For determining hostname
import socket

# Events, for Task
import events

# Subprocess, for NagiosTask
import shlex
import subprocess

# Multiprocessing, for PythonTask
import multiprocessing

# Requests, for CloudKickTask
import requests

# JSON, for JSONTask
import json

# Some module-level constants for definining tasks
DEFAULT_TTL = 60
DEFAULT_HOSTNAME = socket.gethostname()
DEFAULT_MULTIPLIER = 5

SEED_TIMING = 0.5

# Numeric matcher for parsing Nagios performance data
# create class constant compiled regex, to prevent
# unecessarily recompiling this regex string.
NUMERIC_REGEX = re.compile('([\d.]+)')

# Configure logging for the Requests module, decreasing
# the verbosity even when we are running in a more
# verbose mode otherwise.
requests_log = logging.getLogger("requests")
requests_log.setLevel(logging.WARNING)

class Task():
    required_config = ['service', 'arg']

    def __init__(self, config):
        self.config = config

        # Confirm that the passed config object contains required values
        for key in self.required_config:
            if key not in config:
                raise KeyError("Config missing '%s'" % (key))

        self.name = config['service']
        self.arg = config['arg']

        # Handle some common task options and their defaults
        self.ttl_multiplier = config['ttl_multiplier'] if 'ttl_multiplier' in config else DEFAULT_MULTIPLIER
        self.host = config['host'] if 'host' in config else DEFAULT_HOSTNAME
        self.ttl = config['ttl'] if 'ttl' in config else DEFAULT_TTL

        self.attributes = config['attributes'] if 'attributes' in config else {}
        self.tags = set(config['tags']) if 'tags' in config else set()
        self.note = config['note'] if 'note' in config else ""

        # Initialize task internal values
        self.events = []
        self.timings = [SEED_TIMING]
        self.locked = False

        log.info("Creating task: '%s' with TTL of %ss" % (self.name, self.ttl))

    def add_timing(self, value, keep=5):
        log.debug("Task %s took %0.2fs" % (self.name, value))
        self.timings.append(value)
        del self.timings[:-keep]

    def skew(self):
        return sum(self.timings)/len(self.timings)

    def start(self):
        if not self.locked:
            log.info("Starting task: '%s' (usually takes %0.2fs)" % (self.name, self.skew()))
            self.locked = True
            self.start_time = time.time()
            self.run()
        else:
            raise RuntimeError("Task '%s' is locked - cannot start another." % (self.name))

    def drain(self):
        self.join()
        self.end_time = time.time()
        self.add_timing(self.end_time - self.start_time)

        tevents = []
        while len(self.events) > 0:
            tevents.append(self.events.pop(0))

        # Now that we've got our events, free the task to run again
        self.locked = False
        return tevents

class HTTPJSONTask(Task):
    def __init__(self, config):
        Task.__init__(self, config)
        self.q = multiprocessing.Queue()

    def request(self):
        try:
            log.debug("Starting web request to '%s'" % (self.arg))
            resp = requests.get(self.arg)
            self.q.put(resp.json(), timeout=(self.ttl * 0.3))
        except Exception as e:
            log.error("Exception during request method of CloudKickTask '%s'\n%s" % (self.name, str(e)))

    def run(self):
        try:
            self.proc = multiprocessing.Process(target=self.request)
            self.proc.start()
        except Exception as e:
            log.error("Exception starting CloudKickTask '%s'\n%s" % (self.name, str(e)))

    def join(self):
        try:
            json_result = self.q.get(timeout=(self.ttl * 0.3))
            self.proc.join()

            log.debug('CloudKickTask: Processing %s metrics' % (len(json_result['metrics'])))
            for metric in json_result['metrics']:
                note = metric['note'] if 'note' in metric else self.note
                event = events.Event()
                event.ttl = self.ttl * self.ttl_multiplier
                event.host = self.host
                event.tags = self.tags
                event.service = metric['name']
                event.state = metric['state']
                event.metric = metric['value']
                event.description = "%s\nWarn threshold: %s, Error threshold: %s" % (note,
                                                                                metric['warn_threshold'],
                                                                                metric['error_threshold'])
                self.events.append(event)
        except Exception as e:
            log.error("Exception joining CloudKickTask '%s'\n%s" % (self.name, str(e)))

class SubProcessTask(Task):
    def __init__(self, config):
        Task.__init__(self, config)

        self.raw_command = self.arg
        self.command = shlex.split(self.arg)
        self.process = None
        self.use_shell = False
        self.attrprefix = 'task_'

    def run(self):
        try:
            self.process = subprocess.Popen(self.command, stdout=subprocess.PIPE, shell=self.use_shell)
        except Exception as e:
            log.error("Exception running task '%s':\n%s" % (self.name, str(e)))

    def join(self):
        try:
            deadline = self.start_time + (self.ttl * 0.5)
            while deadline > time.time():
                if self.process.poll() == None:
                    time.sleep(0.5)
                else:
                    log.debug("Gathering output from task '%s'" % (self.name))
                    stdout, sterr = self.process.communicate()
                    return stdout, sterr, self.process.returncode
            else:
                log.warning("Deadline expired for task '%s' - force killing" % (self.name))
                self.process.kill()
                self.process.wait()
                log.debug("Subprocess killed for task '%s'" % (self.name))
                return '', '', -127
        except Exception as e:
            log.error("Exception joining task '%s':\n%s" % (self.name, str(e)))

class NagiosTask(SubProcessTask):
    exitcodes = {
     -127: 'timeout',
        0: 'ok',
        1: 'warn',
        2: 'critical',
        3: 'unknown'
    }

    def __init__(self, config):
        SubProcessTask.__init__(self, config)

    def parse_nagios_output(self, stdout):
        # TODO: Be more robust if this fails for any reason
        parts = stdout.split("|")
        if len(parts) == 1:
            log.debug("Task '%s' did not return perf data." % (self.name))
            return (parts[0], None)
        elif len(parts) == 2:
            attributes = {}
            output, raw_perf = parts
            log.debug("Task '%s' returned perf data: %s" % (self.name, raw_perf.strip()))
            for item in raw_perf.strip().split(" "):
                key, val = item.split(';')[0].split('=')
                attributes[self.attrprefix + key] = val

            return output, attributes
        else:
            log.warning("Output for task '%s' could not be parsed for perf data: %s" % (self.name, str(parts)))
            return (stdout, None)

    def join(self):
        try:
            stdout, stderr, returncode = SubProcessTask.join(self)

            event = events.Event()
            event.service = self.name
            event.ttl = self.ttl * self.ttl_multiplier
            event.host = self.host
            event.tags = self.tags
            event.attributes = self.attributes

            output, attributes = self.parse_nagios_output(stdout)
            event.description = "Note: %s\nCommand: %s\nOutput: %s\n" % (self.note, self.raw_command, output)

            if returncode in self.exitcodes:
                event.state = self.exitcodes[returncode]
            else:
                event.state = 'unknown'

            if attributes:
                if 'metric' in config and config['metric'] in attributes:
                    metric_key = config['metric']
                else:
                    metric_key = attributes.keys()[0]

                event.attributes.update(attributes)
                event.metric = float(NUMERIC_REGEX.match(attributes[metric_key]).group(1))

            self.events.append(event)
        except Exception as e:
            log.error("Exception joining task '%s':\n%s" % (self.name, str(e)))

class JSONTask(SubProcessTask):
    def __init__(self, config):
        SubProcessTask.__init__(self, config)

    def join(self):
        try:
            stdout, stderr, returncode = SubProcessTask.join(self)

            try:
                results = json.loads(stdout)
            except Exception as e:
                log.error("Failed to parse JSON. '%s':\n%s" % (self.name, str(e)))

            for result in results:
                for field in ['service', 'state', 'description', 'metric']:
                    if field not in result:
                        log.error("Event missing field '%s'" % (field))
                        continue

                event = events.Event()
                event.ttl = self.ttl * self.ttl_multiplier
                event.host = self.host
                event.tags = self.tags
                event.attributes = self.attributes

                if "attributes" in result:
                    event.attributes.update({self.attrprefix + name: result["attributes"][name] for name in result["attributes"]})

                event.service = result['service']
                event.state = result['state']
                event.metric = result['metric']
                event.description = "%s\n%s" % (self.note, result['description'])
                self.events.append(event)
        except Exception as e:
            log.error("Exception joining task '%s':\n%s" % (self.name, str(e)))
<|endoftext|>