<commit_before>from flask import Flask, render_template, send_file, safe_join, request,jsonify
from threading import Thread
from flask_socketio import SocketIO

import settings
from StandardAnswers import get_answer

import http_calls

app = Flask(__name__)
app.config.from_object('settings')
socketio = SocketIO(app)

@app.route('/', methods=['GET', "POST"])
def index():
    # grammar test
    text = request.args.get('preProc')
    print text
    if text:
        return http_calls.text_preproccess(text)
    else:
        return render_template('index.html')

@app.route('/inter', methods=['GET'])
def interception():
    return render_template('inter.html')

@app.route('/input', methods=["POST"])
def input():
    http_calls.input()

@socketio.on('key')
def handle_my_custom_event(json):
    print('received json: ' + str(json))

def launch():
    def start_websocket_server():

        socketio.run(app, port=app.config['PORT_WEBSOCKET'])
    Thread(target=start_websocket_server).start()
    app.run(debug=app.config['DEBUG'], host=app.config['HOST'], port=app.config['PORT'])

    #logger.debug("Application terminates")

# RUN APP
if __name__ == "__main__":
    launch()
<commit_msg>kicked websockets out again, so its killable<commit_after>from flask import Flask, render_template, send_file, safe_join, request,jsonify
from threading import Thread
from flask_socketio import SocketIO

import settings
from StandardAnswers import get_answer

import http_calls

app = Flask(__name__)
app.config.from_object('settings')
socketio = SocketIO(app)

@app.route('/', methods=['GET', "POST"])
def index():
    # grammar test
    text = request.args.get('preProc')
    print text
    if text:
        return http_calls.text_preproccess(text)
    else:
        return render_template('index.html')

@app.route('/inter', methods=['GET'])
def interception():
    return render_template('inter.html')

@app.route('/input', methods=["POST"])
def input():
    http_calls.input()

@socketio.on('key')
def handle_my_custom_event(json):
    print('received json: ' + str(json))

def launch():
    def start_websocket_server():

        socketio.run(app, port=app.config['PORT_WEBSOCKET'])
    #Thread(target=start_websocket_server).start()
    app.run(debug=app.config['DEBUG'], host=app.config['HOST'], port=app.config['PORT'])

    #logger.debug("Application terminates")

# RUN APP
if __name__ == "__main__":
    launch()
<|endoftext|>


<commit_before>#!/usr/bin/env python2
"""
quattor-pod-doc generates markdown documentation from the
pod's in configuration-modules-core and creates a index for
the website on http://quattor.org.

@author: Wouter Depypere (Ghent University)

"""
import sys
import os
import re
from vsc.utils.generaloption import simple_option
from vsc.utils import fancylogger
from vsc.utils.run import run_asyncloop

MAILREGEX = re.compile(("([a-z0-9!#$%&'*+\/=?^_`{|}~-]+(?:\.[a-z0-9!#$%&'*+\/=?^_`"
                        "{|}~-]+)*(@|\sat\s)(?:[a-z0-9](?:[a-z0-9-]*[a-z0-9])?(\.|"
                        "\sdot\s))+[a-z0-9](?:[a-z0-9-]*[a-z0-9])?)"))
EXAMPLEMAILS = ["example", "username", "system.admin"]
LOGGER = fancylogger.getLogger()

def mavencleancompile(modules_location):
    """
    Executes mvn clean and mvn compile in the given modules_location.
    """

    LOGGER.info("Doing maven clean in %s." % modules_location)
    output = run_asyncloop("mvn clean", startpath=modules_location)
    LOGGER.debug(output)

    LOGGER.info("Doing maven compile in %s." % modules_location)
    output = run_asyncloop("mvn compile", startpath=modules_location)
    LOGGER.debug(output)

def generatemds(pods, location):
    """
    Takes a list of components with podfiles and generates a md file for it.
    """
    LOGGER.info("Generating md files.")
    counter = 0
    mdfiles = []
    for component in sorted(pods):
        for pod in pods[component]:
            title = os.path.splitext(os.path.basename(pod))[0]
            mdfile = "%s-%s.md" % (os.path.join(location, component), title)
            convertpodtomarkdown(pod, mdfile, title)
            counter += 1
            mdfiles.append(mdfile)

    LOGGER.info("Written %s md files." % counter)
    return mdfiles

def convertpodtomarkdown(podfile, outputfile, title):
    """
    Takes a podfile and converts it to a markdown with the help of pod2markdown.
    """
    LOGGER.debug("Running pod2markdown on %s." % podfile)
    output = run_asyncloop("pod2markdown %s" % podfile)
    LOGGER.debug("writing output to %s." % outputfile)
    LOGGER.debug(output)
    fih = open(outputfile, "w")

    fih.write("---\n")
    fih.write("layout: article\n")
    fih.write("title: %s\n" % title)
    fih.write("category: documentation\n")
    fih.write("---\n")

    fih.write(output[1])
    fih.close()

def generatetoc(pods, outputloc, indexname):
    """
    Generates a TOC for the parsed components.
    """
    LOGGER.info("Generating TOC as %s." % os.path.join(outputloc, indexname))

    fih = open(os.path.join(outputloc, indexname), "w")

    fih.write("---\n")
    fih.write("layout: documentation\n")
    fih.write("category: documentation\n")
    fih.write("title: Components\n")
    fih.write("---\n")

    fih.write("\n### Components \n\n")

    for component in sorted(pods):
        if len(pods[component]) == 1:
            linkname = "/documentation/%s-%s.html" % (component, os.path.splitext(os.path.basename(pods[component][0]))[0])
            fih.write(" * [%s](%s) \n" % (component, linkname))
        else:
            fih.write(" * %s \n" % component)
            for pod in pods[component]:
                linkname = "/documentation/%s-%s.html" % (component, os.path.splitext(os.path.basename(pod))[0])
                fih.write("    * [%s](%s) \n" % (os.path.splitext(os.path.basename(pod))[0], linkname))

    fih.write("\n")
    fih.close()

def removemailadresses(mdfiles):
    """
    Removes the email addresses from the markdown files.
    """
    LOGGER.info("Removing emailaddresses from md files.")
    counter = 0
    for mdfile in mdfiles:
        fih = open(mdfile, 'r')
        mdcontent = fih.read()
        fih.close()
        for email in re.findall(MAILREGEX, mdcontent):
            LOGGER.debug("Found %s." % email[0])
            replace = True
            if email[0].startswith('//'):
                replace = False
            for ignoremail in EXAMPLEMAILS:
                if ignoremail in email[0]:
                    replace = False

        if replace:
            LOGGER.debug("Removed it from line.")
            mdcontent = mdcontent.replace(email[0], '')
            fih = open(mdfile, 'w')
            fih.write(mdcontent)
            fih.close()
            counter += 1
    LOGGER.info("Removed %s email addresses." % counter)

def removewhitespace(mdfiles):
    """
    Removes extra whitespace (\n\n\n).
    """
    LOGGER.info("Removing extra whitespace from md files.")
    counter = 0
    for mdfile in mdfiles:
        fih = open(mdfile, 'r')
        mdcontent = fih.read()
        fih.close()
        if '\n\n\n' in mdcontent:
            LOGGER.debug("Removing whitespace in %s." % mdfile)
            mdcontent = mdcontent.replace('\n\n\n', '\n')
            fih = open(mdfile, 'w')
            fih.write(mdcontent)
            fih.close()
            counter += 1
    LOGGER.info("Removed extra whitespace from %s files." % counter)

def decreasetitlesize(mdfiles):
    """
    Makes titles smaller, e.g. replace "# " with "### ".
    """
    LOGGER.info("Downsizing titles in md files.")
    counter = 0
    for mdfile in mdfiles:
        fih = open(mdfile, 'r')
        mdcontent = fih.read()
        fih.close()
        if '# ' in mdcontent:
            LOGGER.debug("Making titles smaller in %s." % mdfile)
            mdcontent = mdcontent.replace('# ', '### ')
            fih = open(mdfile, 'w')
            fih.write(mdcontent)
            fih.close()
            counter += 1
    LOGGER.info("Downsized titles in %s files." % counter)

def removeheaders(mdfiles):
    """
    Removes MAINTAINER and AUTHOR headers from md files.
    """
    LOGGER.info("Removing AUTHOR and MAINTAINER headers from md files.")
    counter = 0
    for mdfile in mdfiles:
        fih = open(mdfile, 'r')
        mdcontent = fih.read()
        fih.close()
        if '# MAINTAINER' in mdcontent:
            LOGGER.debug("Removing # MAINTAINER in %s." % mdfile)
            mdcontent = mdcontent.replace('# MAINTAINER', '')
            fih = open(mdfile, 'w')
            fih.write(mdcontent)
            fih.close()
            counter += 1
        if '# AUTHOR' in mdcontent:
            LOGGER.debug("Removing # AUTHOR in %s." % mdfile)
            mdcontent = mdcontent.replace('# AUTHOR', '')
            fih = open(mdfile, 'w')
            fih.write(mdcontent)
            fih.close()
            counter += 1

    LOGGER.info("Removed %s unused headers." % counter)

def checkinputandcommands(modloc, outputloc, runmaven):
    """
    Check if the directories are in place.
    Check if the required binaries are in place.
    """
    LOGGER.info("Checking if the given paths exist.")
    if not os.path.exists(modloc):
        LOGGER.error("configuration-modules-core location %s does not exist" % modloc)
        sys.exit(1)
    if not os.path.exists(outputloc):
        LOGGER.error("Output location %s does not exist" % outputloc)
        sys.exit(1)

    LOGGER.info("Checking if required binaries are installed.")
    if runmaven:
        if not which("mvn"):
            LOGGER.error("The command mvn is not available on this system, please install maven.")
            sys.exit(1)
    if not which("pod2markdown"):
        LOGGER.error("The command pod2markdown is not available on this system, please install pod2markdown.")
        sys.exit(1)

def listcomponents(module_location):
    """
    return a list of components in module_location.
    """
    components = []
    counter = 0
    LOGGER.info("Searching for components in %s." % module_location)
    for posloc in os.listdir(module_location):
        if os.path.isdir(os.path.join(module_location, posloc)) and posloc.startswith("ncm-"):
            components.append(posloc)
            LOGGER.debug("Adding %s to component list." % posloc)
            counter += 1
    LOGGER.info("Found %s components." % counter)
    return components

def listpods(module_location, components):
    """
    return an array of the pod files per component.
    """
    comppods = {}
    counter = 0
    LOGGER.info("searching for podfiles per component.")
    for comp in components:
        pods = []

        for root, dirs, files in os.walk(os.path.join(module_location, comp, "target")):
            for fileh in files:

                if fileh.endswith(".pod"):
                    LOGGER.debug("%s - %s - %s" % (comp, fileh, root))
                    counter += 1
                    pods.append(os.path.join(root, fileh))

        comppods[comp] = pods
    LOGGER.info("Found %s pod files." % counter)
    LOGGER.debug(comppods)
    return comppods

def which(command):
    """
    Check if given command is available for the current user on this system.
    """
    found = False
    for direct in os.getenv("PATH").split(':'):
        if (os.path.exists(os.path.join(direct, command))):
            found = True

    return found

if __name__ == '__main__':
    OPTIONS = {
        'modules_location': ('The location of the configuration-modules-core checkout.', None, 'store',
                             '/home/wdpypere/workspace/configuration-modules-core', 'm'),
        'output_location': ('The location where the output markdown files should be written to.', None, 'store',
                            '/home/wdpypere/workspace/quattor.github.com/documentation', 'o'),
        'maven_compile': ('Execute a maven clean and maven compile before generating the documentation.', None,
                          'store_true', False, 'c'),
        'index_name': ('Filename for the index/toc for the components', None, 'store', 'components.md', 'i'),
        'remove_emails': ('Remove email addresses from generated md files.', None, 'store_true', True, 'r'),
        'remove_whitespace': ('Remove whitespace (\n\n\n) from md files.', None, 'store_true', True, 'w'),
        'remove_headers': ('Remove unneeded headers from files (MAINTAINER and AUTHOR).', None, 'store_true', True, 'R'),
        'small_titles': ('Decrease the title size in the md files.', None, 'store_true', True, 's')
    }
    GO = simple_option(OPTIONS)
    LOGGER.info("Starting main.")

    checkinputandcommands(GO.options.modules_location, GO.options.output_location, GO.options.maven_compile)

    if GO.options.maven_compile:
        LOGGER.info("Doing maven clean and compile.")
        mavencleancompile(GO.options.modules_location)
    else:
        LOGGER.info("Skipping maven clean and compile.")

    COMPS = listcomponents(GO.options.modules_location)
    PODS = listpods(GO.options.modules_location, COMPS)

    generatetoc(PODS, GO.options.output_location, GO.options.index_name)
    MDS = generatemds(PODS, GO.options.output_location)

    if GO.options.remove_emails:
        removemailadresses(MDS)

    if GO.options.remove_headers:
        removeheaders(MDS)

    if GO.options.small_titles:
        decreasetitlesize(MDS)

    if GO.options.remove_whitespace:
        removewhitespace(MDS)

    LOGGER.info("Done.")
<commit_msg>Remove default locations for modules and output<commit_after>#!/usr/bin/env python2
"""
quattor-pod-doc generates markdown documentation from the
pod's in configuration-modules-core and creates a index for
the website on http://quattor.org.

@author: Wouter Depypere (Ghent University)

"""
import sys
import os
import re
from vsc.utils.generaloption import simple_option
from vsc.utils import fancylogger
from vsc.utils.run import run_asyncloop

MAILREGEX = re.compile(("([a-z0-9!#$%&'*+\/=?^_`{|}~-]+(?:\.[a-z0-9!#$%&'*+\/=?^_`"
                        "{|}~-]+)*(@|\sat\s)(?:[a-z0-9](?:[a-z0-9-]*[a-z0-9])?(\.|"
                        "\sdot\s))+[a-z0-9](?:[a-z0-9-]*[a-z0-9])?)"))
EXAMPLEMAILS = ["example", "username", "system.admin"]
LOGGER = fancylogger.getLogger()

def mavencleancompile(modules_location):
    """
    Executes mvn clean and mvn compile in the given modules_location.
    """

    LOGGER.info("Doing maven clean in %s." % modules_location)
    output = run_asyncloop("mvn clean", startpath=modules_location)
    LOGGER.debug(output)

    LOGGER.info("Doing maven compile in %s." % modules_location)
    output = run_asyncloop("mvn compile", startpath=modules_location)
    LOGGER.debug(output)

def generatemds(pods, location):
    """
    Takes a list of components with podfiles and generates a md file for it.
    """
    LOGGER.info("Generating md files.")
    counter = 0
    mdfiles = []
    for component in sorted(pods):
        for pod in pods[component]:
            title = os.path.splitext(os.path.basename(pod))[0]
            mdfile = "%s-%s.md" % (os.path.join(location, component), title)
            convertpodtomarkdown(pod, mdfile, title)
            counter += 1
            mdfiles.append(mdfile)

    LOGGER.info("Written %s md files." % counter)
    return mdfiles

def convertpodtomarkdown(podfile, outputfile, title):
    """
    Takes a podfile and converts it to a markdown with the help of pod2markdown.
    """
    LOGGER.debug("Running pod2markdown on %s." % podfile)
    output = run_asyncloop("pod2markdown %s" % podfile)
    LOGGER.debug("writing output to %s." % outputfile)
    LOGGER.debug(output)
    fih = open(outputfile, "w")

    fih.write("---\n")
    fih.write("layout: article\n")
    fih.write("title: %s\n" % title)
    fih.write("category: documentation\n")
    fih.write("---\n")

    fih.write(output[1])
    fih.close()

def generatetoc(pods, outputloc, indexname):
    """
    Generates a TOC for the parsed components.
    """
    LOGGER.info("Generating TOC as %s." % os.path.join(outputloc, indexname))

    fih = open(os.path.join(outputloc, indexname), "w")

    fih.write("---\n")
    fih.write("layout: documentation\n")
    fih.write("category: documentation\n")
    fih.write("title: Components\n")
    fih.write("---\n")

    fih.write("\n### Components \n\n")

    for component in sorted(pods):
        if len(pods[component]) == 1:
            linkname = "/documentation/%s-%s.html" % (component, os.path.splitext(os.path.basename(pods[component][0]))[0])
            fih.write(" * [%s](%s) \n" % (component, linkname))
        else:
            fih.write(" * %s \n" % component)
            for pod in pods[component]:
                linkname = "/documentation/%s-%s.html" % (component, os.path.splitext(os.path.basename(pod))[0])
                fih.write("    * [%s](%s) \n" % (os.path.splitext(os.path.basename(pod))[0], linkname))

    fih.write("\n")
    fih.close()

def removemailadresses(mdfiles):
    """
    Removes the email addresses from the markdown files.
    """
    LOGGER.info("Removing emailaddresses from md files.")
    counter = 0
    for mdfile in mdfiles:
        fih = open(mdfile, 'r')
        mdcontent = fih.read()
        fih.close()
        for email in re.findall(MAILREGEX, mdcontent):
            LOGGER.debug("Found %s." % email[0])
            replace = True
            if email[0].startswith('//'):
                replace = False
            for ignoremail in EXAMPLEMAILS:
                if ignoremail in email[0]:
                    replace = False

        if replace:
            LOGGER.debug("Removed it from line.")
            mdcontent = mdcontent.replace(email[0], '')
            fih = open(mdfile, 'w')
            fih.write(mdcontent)
            fih.close()
            counter += 1
    LOGGER.info("Removed %s email addresses." % counter)

def removewhitespace(mdfiles):
    """
    Removes extra whitespace (\n\n\n).
    """
    LOGGER.info("Removing extra whitespace from md files.")
    counter = 0
    for mdfile in mdfiles:
        fih = open(mdfile, 'r')
        mdcontent = fih.read()
        fih.close()
        if '\n\n\n' in mdcontent:
            LOGGER.debug("Removing whitespace in %s." % mdfile)
            mdcontent = mdcontent.replace('\n\n\n', '\n')
            fih = open(mdfile, 'w')
            fih.write(mdcontent)
            fih.close()
            counter += 1
    LOGGER.info("Removed extra whitespace from %s files." % counter)

def decreasetitlesize(mdfiles):
    """
    Makes titles smaller, e.g. replace "# " with "### ".
    """
    LOGGER.info("Downsizing titles in md files.")
    counter = 0
    for mdfile in mdfiles:
        fih = open(mdfile, 'r')
        mdcontent = fih.read()
        fih.close()
        if '# ' in mdcontent:
            LOGGER.debug("Making titles smaller in %s." % mdfile)
            mdcontent = mdcontent.replace('# ', '### ')
            fih = open(mdfile, 'w')
            fih.write(mdcontent)
            fih.close()
            counter += 1
    LOGGER.info("Downsized titles in %s files." % counter)

def removeheaders(mdfiles):
    """
    Removes MAINTAINER and AUTHOR headers from md files.
    """
    LOGGER.info("Removing AUTHOR and MAINTAINER headers from md files.")
    counter = 0
    for mdfile in mdfiles:
        fih = open(mdfile, 'r')
        mdcontent = fih.read()
        fih.close()
        if '# MAINTAINER' in mdcontent:
            LOGGER.debug("Removing # MAINTAINER in %s." % mdfile)
            mdcontent = mdcontent.replace('# MAINTAINER', '')
            fih = open(mdfile, 'w')
            fih.write(mdcontent)
            fih.close()
            counter += 1
        if '# AUTHOR' in mdcontent:
            LOGGER.debug("Removing # AUTHOR in %s." % mdfile)
            mdcontent = mdcontent.replace('# AUTHOR', '')
            fih = open(mdfile, 'w')
            fih.write(mdcontent)
            fih.close()
            counter += 1

    LOGGER.info("Removed %s unused headers." % counter)

def checkinputandcommands(modloc, outputloc, runmaven):
    """
    Check if the directories are in place.
    Check if the required binaries are in place.
    """
    LOGGER.info("Checking if the given paths exist.")
    if not modloc:
        LOGGER.error("configuration-modules-core location not specified")
        sys.exit(1)
    if not outputloc:
        LOGGER.error("output location not specified")
        sys.exit(1)
    if not os.path.exists(modloc):
        LOGGER.error("configuration-modules-core location %s does not exist" % modloc)
        sys.exit(1)
    if not os.path.exists(outputloc):
        LOGGER.error("Output location %s does not exist" % outputloc)
        sys.exit(1)

    LOGGER.info("Checking if required binaries are installed.")
    if runmaven:
        if not which("mvn"):
            LOGGER.error("The command mvn is not available on this system, please install maven.")
            sys.exit(1)
    if not which("pod2markdown"):
        LOGGER.error("The command pod2markdown is not available on this system, please install pod2markdown.")
        sys.exit(1)

def listcomponents(module_location):
    """
    return a list of components in module_location.
    """
    components = []
    counter = 0
    LOGGER.info("Searching for components in %s." % module_location)
    for posloc in os.listdir(module_location):
        if os.path.isdir(os.path.join(module_location, posloc)) and posloc.startswith("ncm-"):
            components.append(posloc)
            LOGGER.debug("Adding %s to component list." % posloc)
            counter += 1
    LOGGER.info("Found %s components." % counter)
    return components

def listpods(module_location, components):
    """
    return an array of the pod files per component.
    """
    comppods = {}
    counter = 0
    LOGGER.info("searching for podfiles per component.")
    for comp in components:
        pods = []

        for root, dirs, files in os.walk(os.path.join(module_location, comp, "target")):
            for fileh in files:

                if fileh.endswith(".pod"):
                    LOGGER.debug("%s - %s - %s" % (comp, fileh, root))
                    counter += 1
                    pods.append(os.path.join(root, fileh))

        comppods[comp] = pods
    LOGGER.info("Found %s pod files." % counter)
    LOGGER.debug(comppods)
    return comppods

def which(command):
    """
    Check if given command is available for the current user on this system.
    """
    found = False
    for direct in os.getenv("PATH").split(':'):
        if (os.path.exists(os.path.join(direct, command))):
            found = True

    return found

if __name__ == '__main__':
    OPTIONS = {
        'modules_location': ('The location of the configuration-modules-core checkout.', None, 'store', None, 'm'),
        'output_location': ('The location where the output markdown files should be written to.', None, 'store', None, 'o'),
        'maven_compile': ('Execute a maven clean and maven compile before generating the documentation.', None, 'store_true', False, 'c'),
        'index_name': ('Filename for the index/toc for the components', None, 'store', 'components.md', 'i'),
        'remove_emails': ('Remove email addresses from generated md files.', None, 'store_true', True, 'r'),
        'remove_whitespace': ('Remove whitespace (\n\n\n) from md files.', None, 'store_true', True, 'w'),
        'remove_headers': ('Remove unneeded headers from files (MAINTAINER and AUTHOR).', None, 'store_true', True, 'R'),
        'small_titles': ('Decrease the title size in the md files.', None, 'store_true', True, 's')
    }
    GO = simple_option(OPTIONS)
    LOGGER.info("Starting main.")

    checkinputandcommands(GO.options.modules_location, GO.options.output_location, GO.options.maven_compile)

    if GO.options.maven_compile:
        LOGGER.info("Doing maven clean and compile.")
        mavencleancompile(GO.options.modules_location)
    else:
        LOGGER.info("Skipping maven clean and compile.")

    COMPS = listcomponents(GO.options.modules_location)
    PODS = listpods(GO.options.modules_location, COMPS)

    generatetoc(PODS, GO.options.output_location, GO.options.index_name)
    MDS = generatemds(PODS, GO.options.output_location)

    if GO.options.remove_emails:
        removemailadresses(MDS)

    if GO.options.remove_headers:
        removeheaders(MDS)

    if GO.options.small_titles:
        decreasetitlesize(MDS)

    if GO.options.remove_whitespace:
        removewhitespace(MDS)

    LOGGER.info("Done.")
<|endoftext|>


<commit_before>#  Copyright 2008-2013 Nokia Siemens Networks Oyj
#
#  Licensed under the Apache License, Version 2.0 (the "License");
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.
from __future__ import with_statement

import os
import signal
import subprocess
import sys
import tempfile

from robot.utils import ConnectionCache
from robot.version import get_version

class Process(object):
    """Robot Framework test library for running processes.

    This library utilizes Python's
    [http://docs.python.org/2.7/library/subprocess.html#subprocess.Popen|Subprocess API and its Popen class].

    The library has following main usages:

    - Starting a processes, and managing their handles, stdouts and stderrs
    - Stopping processes started by this library
    - Switching between processes
    - Checking process status.

    == Table of contents ==

    - `Configurations`
    - `Example`
    - `Active process`
    - `ExecutionResult`

    = Configurations =
    `Run Process` and `Start New Process` keywords can be given several named
    arguments, which are listed below. Default values in the parenthesis.

     - `cwd` specifies if program's directory is used as cwd  (`False`)
     - `shell` specifies if shell is used for program execution (`False`)
     - `stdout` is a file handle for standard output
     - `stderr` is a file handle for standard error
     - `stdin` is a file handle for standard input
     - `alias` is a name for the process (`None`).

    = Example =
    The following example demonstrates library's main usages as stated above.

    | *** Settings *** |  |  |  |  |
    | Library | Process |  |  |  |
    |  |  |  |  |  |
    | *** Test Cases *** |  |  |  |  |
    | Example |  |  |  |  |
    | ${handle1}= | `Start New Process` | /path/command.sh    | shell=True  | cwd=/path |
    | ${handle2}= | `Start New Process` | ${CURDIR}${/}mytool   | shell=True |   |
    | ${result1}=  | `Wait For Process` | ${handle1} |    |  |
    |  | `Terminate Process` | ${handle2} |    |   |
    |  | `Process Should Be Dead` | ${handle2} |    |   |
    |  | [Teardown] | `Kill All Processes` |    |   |

    = Active process =
    Many of the library keywords have `handle` as optional argument. This means
    that if argument `handle` is NOT given, then current active process is used.

    = ExecutionResult =
    This object contains information about the process execution.

    Included information is:

    - `stdout` standard output file handle
    - `stderr` standard error file handle
    - `exit_code` from the process, `None` during execution.
    """

    ROBOT_LIBRARY_SCOPE='GLOBAL'
    ROBOT_LIBRARY_VERSION = get_version()

    def __init__(self):
        self._started_processes = ConnectionCache()
        self._logs = dict()
        self._tempdir = tempfile.mkdtemp(suffix="processlib")

    def run_process(self, command, *args, **conf):
        """This keyword runs a process and waits for it to terminate.

        The `command` is a child program which is started in a new process,
        `args` are arguments for the `command` and `conf` are arguments for the
        Subprocess API (see `Configurations`).

        Finally it switches back to active process.
        """
        active_process_index = self._started_processes.current_index
        try:
            p = self.start_new_process(command, *args, **conf)
            return self.wait_for_process(p)
        finally:
            self._started_processes.switch(active_process_index)

    def start_new_process(self, command, *args, **conf):
        """This keyword starts a new process.

        The `command` is a child program which is started in a new process,
        `args` are arguments for the `command` and `conf` are arguments for the
        Subprocess API (see `Configurations`).

        Returns process index on success.

        Examples:

        | $handle1}= | `Start New Process` | /bin/script.sh | |
        | $handle2}= | `Start New Process` | totals | |
        """
        config = NewProcessConfig(self._tempdir, **conf)
        p = subprocess.Popen(self._cmd(args, command, config.shell),
                             stdout=config.stdout_stream,
                             stderr=config.stderr_stream,
                             stdin=subprocess.PIPE,
                             shell=config.shell,
                             cwd=config.cwd)
        self._logs[p] = ExecutionResult(config.stdout_stream.name,
                                        config.stderr_stream.name)
        return self._started_processes.register(p, alias=config.alias)

    def _cmd(self, args, command, use_shell):
        cmd = [command] + [str(i) for i in args]
        if use_shell and args:
            cmd = subprocess.list2cmdline(cmd)
        elif use_shell:
            cmd = command
        return cmd

    def process_is_alive(self, handle=None):
        """Returns a running status of process with `handle`. Argument `handle`
        is optional, if `None` then the current process is used.

        Return value is either `True` or `False`
        """
        return self._process(handle).poll() is None

    def process_should_be_alive(self, handle=None):
        """Assertion keyword, which expects that process with `handle` is alive.
        Argument `handle` is optional, if `None` then the current process is used.

        Check is done using `Process Is Alive` keyword.

        Raises an error if process is dead.
        """
        if not self.process_is_alive(handle):
            raise AssertionError('Process is not alive')

    def process_should_be_dead(self, handle=None):
        """Assertion keyword, which expects that process with `handle` is dead.
        Argument `handle` is optional, if left then the current process is used.

        Check is done using `Process Is Alive` keyword.

        Raises an error if process is alive.
        """
        if self.process_is_alive(handle):
            raise AssertionError('Process is alive')

    def wait_for_process(self, handle=None):
        """This waits for process with `handle` to complete its execution.

        Argument `handle` is optional, if left then the current process is used.

        Returns an `ExecutionResult` object.

        Examples:

        | ${output}= | `Wait For Process` | |
        | Should Be Equal As Integers | ${output.exit_code} | 0 |
        | Should Match | ${output.stdout} | `*text in the out*` |
        | Should Match | ${output.stderr} | |
        """
        process = self._process(handle)
        result = self._logs[process]
        result.exit_code = process.wait()
        return result

    def terminate_process(self, handle=None, kill=False):
        """This keyword terminates process using either `Subprocess` `kill()`
        or `terminate()`, which can be selected using `kill` argument.

        Argument `handle` is optional, if `None` then the current process is used.

        Examples:

        | `Terminate Process` | | # Terminates the current process |
        | `Terminate Process` | ${handle3} | |
        """
        process = self._process(handle)

        # This should be enough to check if we are dealing with <2.6 Python
        if not hasattr(process,'kill'):
            self._terminate_process(process)
            return
        if kill:
            process.kill()
        else:
            process.terminate()

    def _terminate_process(self, theprocess):
        if sys.platform == 'win32':
            import ctypes
            PROCESS_TERMINATE = 1
            handle = ctypes.windll.kernel32.OpenProcess(PROCESS_TERMINATE,
                                                        False,
                                                        theprocess.pid)
            ctypes.windll.kernel32.TerminateProcess(handle, -1)
            ctypes.windll.kernel32.CloseHandle(handle)
        else:
            os.kill(theprocess.pid, signal.SIGKILL)

    def kill_all_processes(self):
        """This keyword terminates all processes started by the library.
        """
        for handle in range(len(self._started_processes._connections)):
            if self.process_is_alive(handle):
                self.terminate_process(handle, kill=True)

    def get_process_id(self, handle=None):
        """Returns a process ID of process with `handle`.

        Argument `handle` is optional, if `None` then the current process is used.

        Return value is a integer value.

        Examples:

        | ${pid}= | `Get Process Id` | | # Gets PID of the active process | |
        | ${handle1}= | `Start New Process` | python -c "print 'hello'" | shell=True | alias=hello |
        | ${pid_1}= | `Get Process Id` | ${handle1} | # Gets PID with `handle1` | |
        | ${pid_2}= | `Get Process Id` | ${handle1} | # Gets PID with alias `hello` | |
        | Should Be Equal As Integers | ${pid_1} | ${pid_2} |  | |
        """
        return self._process(handle).pid

    def input_to_process(self, msg, handle=None):
        """This keyword can be used to communicate with the process with `handle`.

        Argument `handle` is optional, if `None` then the current process is used.

        Examples:

        | `Wait For Process` | ${handle3} | |
        | `Input To Process` | ${PASSWORD} | ${handle3} |
        """
        self._process(handle).communicate(msg+'\n')

    def switch_active_process(self, handle):
        """This keyword switches active process into process with `handle`.

        Examples:

        | Run Keyword And Expect Error | `Switch Active Process` | |
        | Run Keyword And Expect Error | `Switch Active Process` | ${nonexistent_handle} |
        | `Switch Active Process` | ${handle1} | |
        """
        self._started_processes.switch(handle)

    def _process(self, handle):
        if handle:
            process,_ = self._started_processes.get_connection(handle)
        else:
            process = self._started_processes.current
        return process

class ExecutionResult(object):

    _stdout = _stderr = None

    def __init__(self, stdout_name, stderr_name, exit_code=None):
        self._stdout_name = stdout_name
        self._stderr_name = stderr_name
        self.exit_code = exit_code

    @property
    def stdout(self):
        if self._stdout is None:
            with open(self._stdout_name,'r') as f:
                self._stdout = f.read()
        return self._stdout

    @property
    def stderr(self):
        if self._stderr is None:
            with open(self._stderr_name,'r') as f:
                self._stderr = f.read()
        return self._stderr

    def __str__(self):
        return """\
stdout_name : %s
stderr_name : %s
exit_code   : %d""" % (self._stdout_name, self._stderr_name, self.exit_code)

class NewProcessConfig(object):

    FILE_INDEX = 0

    def __init__(self, tempdir, cwd=None,
                 shell=False, stdout=None,
                 stderr=None, alias=None):
        self._tempdir = tempdir
        self.cwd = cwd or os.path.abspath(os.curdir)
        self.stdout_stream = self._new_stream(stdout, 'stdout')
        self.stderr_stream = self._get_stderr(stderr, stdout)
        self.shell = bool(shell)
        self.alias = alias

    def _new_stream(self, name, postfix):
        if name:
            return open(os.path.join(self.cwd, name), 'w')
        return self._get_temp_file(postfix)

    def _get_stderr(self, stderr, stdout):
        if stderr:
            if stderr == 'STDOUT' or stderr == stdout:
               return self.stdout_stream
        return self._new_stream(stderr, 'stderr')

    def _get_temp_file(self, suffix):
        filename = 'tmp_logfile_%d_%s.out' % (NewProcessConfig.FILE_INDEX, suffix)
        NewProcessConfig.FILE_INDEX += 1
        return open(os.path.join(self._tempdir, filename), 'w')
<commit_msg>Process: kill None Pid<commit_after>#  Copyright 2008-2013 Nokia Siemens Networks Oyj
#
#  Licensed under the Apache License, Version 2.0 (the "License");
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.
from __future__ import with_statement

import os
import signal
import subprocess
import sys
import tempfile

from robot.utils import ConnectionCache
from robot.version import get_version

class Process(object):
    """Robot Framework test library for running processes.

    This library utilizes Python's
    [http://docs.python.org/2.7/library/subprocess.html#subprocess.Popen|Subprocess API and its Popen class].

    The library has following main usages:

    - Starting a processes, and managing their handles, stdouts and stderrs
    - Stopping processes started by this library
    - Switching between processes
    - Checking process status.

    == Table of contents ==

    - `Configurations`
    - `Example`
    - `Active process`
    - `ExecutionResult`

    = Configurations =
    `Run Process` and `Start New Process` keywords can be given several named
    arguments, which are listed below. Default values in the parenthesis.

     - `cwd` specifies if program's directory is used as cwd  (`False`)
     - `shell` specifies if shell is used for program execution (`False`)
     - `stdout` is a file handle for standard output
     - `stderr` is a file handle for standard error
     - `stdin` is a file handle for standard input
     - `alias` is a name for the process (`None`).

    = Example =
    The following example demonstrates library's main usages as stated above.

    | *** Settings *** |  |  |  |  |
    | Library | Process |  |  |  |
    |  |  |  |  |  |
    | *** Test Cases *** |  |  |  |  |
    | Example |  |  |  |  |
    | ${handle1}= | `Start New Process` | /path/command.sh    | shell=True  | cwd=/path |
    | ${handle2}= | `Start New Process` | ${CURDIR}${/}mytool   | shell=True |   |
    | ${result1}=  | `Wait For Process` | ${handle1} |    |  |
    |  | `Terminate Process` | ${handle2} |    |   |
    |  | `Process Should Be Dead` | ${handle2} |    |   |
    |  | [Teardown] | `Kill All Processes` |    |   |

    = Active process =
    Many of the library keywords have `handle` as optional argument. This means
    that if argument `handle` is NOT given, then current active process is used.

    = ExecutionResult =
    This object contains information about the process execution.

    Included information is:

    - `stdout` standard output file handle
    - `stderr` standard error file handle
    - `exit_code` from the process, `None` during execution.
    """

    ROBOT_LIBRARY_SCOPE='GLOBAL'
    ROBOT_LIBRARY_VERSION = get_version()

    def __init__(self):
        self._started_processes = ConnectionCache()
        self._logs = dict()
        self._tempdir = tempfile.mkdtemp(suffix="processlib")

    def run_process(self, command, *args, **conf):
        """This keyword runs a process and waits for it to terminate.

        The `command` is a child program which is started in a new process,
        `args` are arguments for the `command` and `conf` are arguments for the
        Subprocess API (see `Configurations`).

        Finally it switches back to active process.
        """
        active_process_index = self._started_processes.current_index
        try:
            p = self.start_new_process(command, *args, **conf)
            return self.wait_for_process(p)
        finally:
            self._started_processes.switch(active_process_index)

    def start_new_process(self, command, *args, **conf):
        """This keyword starts a new process.

        The `command` is a child program which is started in a new process,
        `args` are arguments for the `command` and `conf` are arguments for the
        Subprocess API (see `Configurations`).

        Returns process index on success.

        Examples:

        | $handle1}= | `Start New Process` | /bin/script.sh | |
        | $handle2}= | `Start New Process` | totals | |
        """
        config = NewProcessConfig(self._tempdir, **conf)
        p = subprocess.Popen(self._cmd(args, command, config.shell),
                             stdout=config.stdout_stream,
                             stderr=config.stderr_stream,
                             stdin=subprocess.PIPE,
                             shell=config.shell,
                             cwd=config.cwd)
        self._logs[p] = ExecutionResult(config.stdout_stream.name,
                                        config.stderr_stream.name)
        return self._started_processes.register(p, alias=config.alias)

    def _cmd(self, args, command, use_shell):
        cmd = [command] + [str(i) for i in args]
        if use_shell and args:
            cmd = subprocess.list2cmdline(cmd)
        elif use_shell:
            cmd = command
        return cmd

    def process_is_alive(self, handle=None):
        """Returns a running status of process with `handle`. Argument `handle`
        is optional, if `None` then the current process is used.

        Return value is either `True` or `False`
        """
        return self._process(handle).poll() is None

    def process_should_be_alive(self, handle=None):
        """Assertion keyword, which expects that process with `handle` is alive.
        Argument `handle` is optional, if `None` then the current process is used.

        Check is done using `Process Is Alive` keyword.

        Raises an error if process is dead.
        """
        if not self.process_is_alive(handle):
            raise AssertionError('Process is not alive')

    def process_should_be_dead(self, handle=None):
        """Assertion keyword, which expects that process with `handle` is dead.
        Argument `handle` is optional, if left then the current process is used.

        Check is done using `Process Is Alive` keyword.

        Raises an error if process is alive.
        """
        if self.process_is_alive(handle):
            raise AssertionError('Process is alive')

    def wait_for_process(self, handle=None):
        """This waits for process with `handle` to complete its execution.

        Argument `handle` is optional, if left then the current process is used.

        Returns an `ExecutionResult` object.

        Examples:

        | ${output}= | `Wait For Process` | |
        | Should Be Equal As Integers | ${output.exit_code} | 0 |
        | Should Match | ${output.stdout} | `*text in the out*` |
        | Should Match | ${output.stderr} | |
        """
        process = self._process(handle)
        result = self._logs[process]
        result.exit_code = process.wait()
        return result

    def terminate_process(self, handle=None, kill=False):
        """This keyword terminates process using either `Subprocess` `kill()`
        or `terminate()`, which can be selected using `kill` argument.

        Argument `handle` is optional, if `None` then the current process is used.

        Examples:

        | `Terminate Process` | | # Terminates the current process |
        | `Terminate Process` | ${handle3} | |
        """
        process = self._process(handle)

        # This should be enough to check if we are dealing with <2.6 Python
        if not hasattr(process,'kill'):
            self._terminate_process(process)
            return
        if kill:
            process.kill()
        else:
            process.terminate()

    def _terminate_process(self, theprocess):
        if sys.platform == 'win32':
            import ctypes
            PROCESS_TERMINATE = 1
            handle = ctypes.windll.kernel32.OpenProcess(PROCESS_TERMINATE,
                                                        False,
                                                        theprocess.pid)
            ctypes.windll.kernel32.TerminateProcess(handle, -1)
            ctypes.windll.kernel32.CloseHandle(handle)
        else:
            pid = theprocess.pid
            if pid is not None:
                os.kill(theprocess.pid, signal.SIGKILL)
            else:
                raise AssertionError('None Pid - can not kill process!')

    def kill_all_processes(self):
        """This keyword terminates all processes started by the library.
        """
        for handle in range(len(self._started_processes._connections)):
            if self.process_is_alive(handle):
                self.terminate_process(handle, kill=True)

    def get_process_id(self, handle=None):
        """Returns a process ID of process with `handle`.

        Argument `handle` is optional, if `None` then the current process is used.

        Return value is a integer value.

        Examples:

        | ${pid}= | `Get Process Id` | | # Gets PID of the active process | |
        | ${handle1}= | `Start New Process` | python -c "print 'hello'" | shell=True | alias=hello |
        | ${pid_1}= | `Get Process Id` | ${handle1} | # Gets PID with `handle1` | |
        | ${pid_2}= | `Get Process Id` | ${handle1} | # Gets PID with alias `hello` | |
        | Should Be Equal As Integers | ${pid_1} | ${pid_2} |  | |
        """
        return self._process(handle).pid

    def input_to_process(self, msg, handle=None):
        """This keyword can be used to communicate with the process with `handle`.

        Argument `handle` is optional, if `None` then the current process is used.

        Examples:

        | `Wait For Process` | ${handle3} | |
        | `Input To Process` | ${PASSWORD} | ${handle3} |
        """
        self._process(handle).communicate(msg+'\n')

    def switch_active_process(self, handle):
        """This keyword switches active process into process with `handle`.

        Examples:

        | Run Keyword And Expect Error | `Switch Active Process` | |
        | Run Keyword And Expect Error | `Switch Active Process` | ${nonexistent_handle} |
        | `Switch Active Process` | ${handle1} | |
        """
        self._started_processes.switch(handle)

    def _process(self, handle):
        if handle:
            process,_ = self._started_processes.get_connection(handle)
        else:
            process = self._started_processes.current
        return process

class ExecutionResult(object):

    _stdout = _stderr = None

    def __init__(self, stdout_name, stderr_name, exit_code=None):
        self._stdout_name = stdout_name
        self._stderr_name = stderr_name
        self.exit_code = exit_code

    @property
    def stdout(self):
        if self._stdout is None:
            with open(self._stdout_name,'r') as f:
                self._stdout = f.read()
        return self._stdout

    @property
    def stderr(self):
        if self._stderr is None:
            with open(self._stderr_name,'r') as f:
                self._stderr = f.read()
        return self._stderr

    def __str__(self):
        return """\
stdout_name : %s
stderr_name : %s
exit_code   : %d""" % (self._stdout_name, self._stderr_name, self.exit_code)

class NewProcessConfig(object):

    FILE_INDEX = 0

    def __init__(self, tempdir, cwd=None,
                 shell=False, stdout=None,
                 stderr=None, alias=None):
        self._tempdir = tempdir
        self.cwd = cwd or os.path.abspath(os.curdir)
        self.stdout_stream = self._new_stream(stdout, 'stdout')
        self.stderr_stream = self._get_stderr(stderr, stdout)
        self.shell = bool(shell)
        self.alias = alias

    def _new_stream(self, name, postfix):
        if name:
            return open(os.path.join(self.cwd, name), 'w')
        return self._get_temp_file(postfix)

    def _get_stderr(self, stderr, stdout):
        if stderr:
            if stderr == 'STDOUT' or stderr == stdout:
               return self.stdout_stream
        return self._new_stream(stderr, 'stderr')

    def _get_temp_file(self, suffix):
        filename = 'tmp_logfile_%d_%s.out' % (NewProcessConfig.FILE_INDEX, suffix)
        NewProcessConfig.FILE_INDEX += 1
        return open(os.path.join(self._tempdir, filename), 'w')
<|endoftext|>


<commit_before>from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals

import err

from plotting_abstract import PlottingBase

def factory(lib_name, *args):
    # Matplotlib
    if lib_name == 'mp':
        import plotMP
        plotting = plotMP.Plotting(*args)
    # pyQtGraph
    elif lib_name == 'pg':
        import plotPG
        plotting = plotPG.Plotting(*args)
    # Bokeh
    elif lib_name == 'bk':
        import plotBK
        raise NotImplementedError
    # Plotting disabled
    elif lib_name is None:
        plotting = PlottingDisabled()
    else:
        raise err.Fatal('unknown plotting library requested: {}'.format(lib_name))

    assert(isinstance(plotting, PlottingBase))
    return plotting

class PlottingDisabled(PlottingBase):
    def figure(self):
        return

    def show(self):
        return

    def plot_rect(self, r, edgecolor='k'):
        return

    def plot_abs_states(self, AA, s):
        return

    def plot_trace_list(self, trace_list, x_vs_y=None):
        return

    def plot_pwa_traces(self, txs):
        return

    def plot(self, *args):
        return

def plot_opts_parse(popts):
    if popts:
        xVsy = popts.split('-')
        return xVsy
    else:
        return []
<commit_msg>abs import fix<commit_after>from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals

import err

from .plotting_abstract import PlottingBase

def factory(lib_name, *args):
    # Matplotlib
    if lib_name == 'mp':
        from . import plotMP
        plotting = plotMP.Plotting(*args)
    # pyQtGraph
    elif lib_name == 'pg':
        from . import plotPG
        plotting = plotPG.Plotting(*args)
    # Bokeh
    elif lib_name == 'bk':
        from . import plotBK
        raise NotImplementedError
    # Plotting disabled
    elif lib_name is None:
        plotting = PlottingDisabled()
    else:
        raise err.Fatal('unknown plotting library requested: {}'.format(lib_name))

    assert(isinstance(plotting, PlottingBase))
    return plotting

class PlottingDisabled(PlottingBase):
    def figure(self):
        return

    def show(self):
        return

    def plot_rect(self, r, edgecolor='k'):
        return

    def plot_abs_states(self, AA, s):
        return

    def plot_trace_list(self, trace_list, x_vs_y=None):
        return

    def plot_pwa_traces(self, txs):
        return

    def plot(self, *args):
        return

def plot_opts_parse(popts):
    if popts:
        xVsy = popts.split('-')
        return xVsy
    else:
        return []
<|endoftext|>


<commit_before>"""
Created on 25 Oct 2017

@author: Bruno Beloff (bruno.beloff@southcoastscience.com)
"""

from collections import OrderedDict
from multiprocessing import Manager

from scs_core.data.average import Average

from scs_core.position.nmea.gpgga import GPGGA
from scs_core.position.gps_datum import GPSDatum

from scs_core.sync.interval_timer import IntervalTimer
from scs_core.sync.synchronised_process import SynchronisedProcess

# --------------------------------------------------------------------------------------------------------------------

class GPSMonitor(SynchronisedProcess):
    """
    classdocs
    """

    # ----------------------------------------------------------------------------------------------------------------

    def __init__(self, gps, conf):
        """
        Constructor
        """
        manager = Manager()

        SynchronisedProcess.__init__(self, manager.list())

        self.__gps = gps
        self.__sample_interval = conf.sample_interval
        self.__averaging = Average(conf.tally)

    # ----------------------------------------------------------------------------------------------------------------
    # SynchronisedProcess implementation...

    def start(self):
        try:
            self.__gps.power_on()
            self.__gps.open()

            self.__averaging.reset()

            super().start()

        except KeyboardInterrupt:
            pass

    def stop(self):
        try:
            super().stop()

            self.__gps.close()

        except KeyboardInterrupt:
            pass

    def run(self):
        # default report...
        datum = GPSDatum.construct_null()

        with self._lock:
            datum.as_list(self._value)

        # sample...
        try:
            timer = IntervalTimer(self.__sample_interval)

            while timer.true():
                # position...
                gga = self.__gps.report(GPGGA)
                datum = GPSDatum.construct_from_gga(gga)

                if datum is None:
                    continue

                # average...
                if datum.quality > 0:
                    self.__averaging.append(datum)          # only append valid positional fixes

                average = self.__averaging.compute()

                if average is None:
                    average = datum                         # provide current datum when there is no average

                # report...
                with self._lock:
                    average.as_list(self._value)

        except KeyboardInterrupt:
            pass

    # ----------------------------------------------------------------------------------------------------------------
    # data retrieval for client process...

    def sample(self):
        with self._lock:
            value = self._value

        return GPSDatum.construct_from_jdict(OrderedDict(value))

    # ----------------------------------------------------------------------------------------------------------------

    def __str__(self, *args, **kwargs):
        return "GPSMonitor:{value:%s, sample_interval:%s, averaging:%s, gps:%s}" % \
               (self._value, self.__sample_interval, self.__averaging, self.__gps)
<commit_msg>GPSMonitor now returns null datum instead of None.<commit_after>"""
Created on 25 Oct 2017

@author: Bruno Beloff (bruno.beloff@southcoastscience.com)
"""

from collections import OrderedDict
from multiprocessing import Manager

from scs_core.data.average import Average

from scs_core.position.nmea.gpgga import GPGGA
from scs_core.position.gps_datum import GPSDatum

from scs_core.sync.interval_timer import IntervalTimer
from scs_core.sync.synchronised_process import SynchronisedProcess

# --------------------------------------------------------------------------------------------------------------------

class GPSMonitor(SynchronisedProcess):
    """
    classdocs
    """

    # ----------------------------------------------------------------------------------------------------------------

    def __init__(self, gps, conf):
        """
        Constructor
        """
        manager = Manager()

        SynchronisedProcess.__init__(self, manager.list())

        self.__gps = gps
        self.__sample_interval = conf.sample_interval
        self.__averaging = Average(conf.tally)

    # ----------------------------------------------------------------------------------------------------------------
    # SynchronisedProcess implementation...

    def start(self):
        try:
            self.__gps.power_on()
            self.__gps.open()

            self.__averaging.reset()

            super().start()

        except KeyboardInterrupt:
            pass

    def stop(self):
        try:
            super().stop()

            self.__gps.close()

        except KeyboardInterrupt:
            pass

    def run(self):
        # default report...
        datum = GPSDatum.null_datum()

        with self._lock:
            datum.as_list(self._value)

        # sample...
        try:
            timer = IntervalTimer(self.__sample_interval)

            while timer.true():
                # position...
                gga = self.__gps.report(GPGGA)
                datum = GPSDatum.construct_from_gga(gga)

                if datum is None:
                    continue

                # average...
                if datum.quality > 0:
                    self.__averaging.append(datum)          # only append valid positional fixes

                average = self.__averaging.compute()

                if average is None:
                    average = datum                         # provide current datum when there is no average

                # report...
                with self._lock:
                    average.as_list(self._value)

        except KeyboardInterrupt:
            pass

    # ----------------------------------------------------------------------------------------------------------------
    # data retrieval for client process...

    def sample(self):
        with self._lock:
            value = self._value

        return GPSDatum.construct_from_jdict(OrderedDict(value))

    # ----------------------------------------------------------------------------------------------------------------

    def __str__(self, *args, **kwargs):
        return "GPSMonitor:{value:%s, sample_interval:%s, averaging:%s, gps:%s}" % \
               (self._value, self.__sample_interval, self.__averaging, self.__gps)
<|endoftext|>


<commit_before>#!/usr/bin/env python3

"""
Created on 21 Sep 2020

@author: Jade Page (jade.page@southcoastscience.com)

DESCRIPTION The aws_group_setup utility is designed to automate the creation of AWS Greengrass groups using South
Coast Science's configuration

The group must already exist and the ML lambdas must be associated with the greengrass account for which the IAM auth
keys are given

SYNOPSIS
aws_group_setup.py [{  [-c] | [-s] } [-m] [-a AWS_Group_Name]] [-v]

EXAMPLES
./aws_group_setup.py -s -a scs-test-001-group -m

FILES
A conf file is placed in a default directory referencing the group name and when it was created

https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/greengrass.html
"""

import boto3
import socket
import sys
from botocore.exceptions import ClientError

from getpass import getpass

from scs_core.aws.greengrass.aws_group import AWSGroup
from scs_core.aws.greengrass.aws_group_configurator import AWSGroupConfigurator
from scs_core.aws.greengrass.gg_errors import ProjectMissingError

from scs_core.data.json import JSONify

from scs_host.sys.host import Host

from scs_mfr.cmd.cmd_aws_group_setup import CmdAWSGroupSetup

# --------------------------------------------------------------------------------------------------------------------

def create_aws_client():
    access_key_secret = ""
    access_key_id = input("Enter AWS Access Key ID or leave blank to use environment variables: ")
    if access_key_id:
        access_key_secret = getpass(prompt="Enter Secret AWS Access Key: ")

    if access_key_id and access_key_secret:
        client = boto3.client(
            'greengrass',
            aws_access_key_id=access_key_id,
            aws_secret_access_key=access_key_secret,
            region_name='us-west-2'
        )
    else:
        client = boto3.client('greengrass', region_name=aws_region)

    return client

def return_group_name():
    host_name = socket.gethostname()
    return host_name + "-group"

# --------------------------------------------------------------------------------------------------------------------

if __name__ == '__main__':

    # ----------------------------------------------------------------------------------------------------------------
    # cmd...

    cmd = CmdAWSGroupSetup()

    if not cmd.is_valid():
        cmd.print_help(sys.stderr)
        exit(2)

    if cmd.verbose:
        print("aws_group_setup: %s" % cmd, file=sys.stderr)
        sys.stderr.flush()

    # ----------------------------------------------------------------------------------------------------------------
    # resources...

    aws_group_name = cmd.aws_group_name if cmd.aws_group_name else return_group_name()
    aws_region = "us-west-2"

    # ----------------------------------------------------------------------------------------------------------------
    # run...

    # ClientAuth...
    awsGroupConf = AWSGroupConfigurator.load(Host)

    if cmd.set:
        if awsGroupConf:
            user_choice = input("Group configuration already exists. Type Yes to update: ")
            print("")
            if not user_choice.lower() == "yes":
                print("Operation cancelled")
                exit()
        try:
            aws_configurator = AWSGroupConfigurator(aws_group_name, create_aws_client(), cmd.use_ml)

            aws_configurator.collect_information(Host)
            aws_configurator.define_aws_group_resources(Host)
            aws_configurator.define_aws_group_functions()
            aws_configurator.define_aws_group_subscriptions()
            aws_configurator.create_aws_group_definition()
            aws_configurator.save(Host)

        except ClientError as error:
            if error.response['Error']['Code'] == 'BadRequestException':
                print("aws_group_setup: Invalid request.", file=sys.stderr)
            if error.response['Error']['Code'] == 'InternalServerErrorException':
                print("aws_group_setup: AWS server error.", file=sys.stderr)

        except ProjectMissingError:
            print("aws_group_setup: Project configuration not set.", file=sys.stderr)

    if cmd.show_current:
        aws_group_info = AWSGroup(cmd.aws_group_name, create_aws_client())

        aws_group_info.get_group_info_from_name()
        aws_group_info.get_group_arns()
        aws_group_info.output_current_info()

        print(JSONify.dumps(aws_group_info))
<commit_msg>Added try catch for key error in case group configuration doesn't exist<commit_after>#!/usr/bin/env python3

"""
Created on 21 Sep 2020

@author: Jade Page (jade.page@southcoastscience.com)

DESCRIPTION The aws_group_setup utility is designed to automate the creation of AWS Greengrass groups using South
Coast Science's configuration

The group must already exist and the ML lambdas must be associated with the greengrass account for which the IAM auth
keys are given

SYNOPSIS
aws_group_setup.py [{  [-c] | [-s] } [-m] [-a AWS_Group_Name]] [-v]

EXAMPLES
./aws_group_setup.py -s -a scs-test-001-group -m

FILES
A conf file is placed in a default directory referencing the group name and when it was created

https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/greengrass.html
"""

import boto3
import socket
import sys
from botocore.exceptions import ClientError

from getpass import getpass

from scs_core.aws.greengrass.aws_group import AWSGroup
from scs_core.aws.greengrass.aws_group_configurator import AWSGroupConfigurator
from scs_core.aws.greengrass.gg_errors import ProjectMissingError

from scs_core.data.json import JSONify

from scs_host.sys.host import Host

from scs_mfr.cmd.cmd_aws_group_setup import CmdAWSGroupSetup

# --------------------------------------------------------------------------------------------------------------------

def create_aws_client():
    access_key_secret = ""
    access_key_id = input("Enter AWS Access Key ID or leave blank to use environment variables: ")
    if access_key_id:
        access_key_secret = getpass(prompt="Enter Secret AWS Access Key: ")

    if access_key_id and access_key_secret:
        client = boto3.client(
            'greengrass',
            aws_access_key_id=access_key_id,
            aws_secret_access_key=access_key_secret,
            region_name='us-west-2'
        )
    else:
        client = boto3.client('greengrass', region_name=aws_region)

    return client

def return_group_name():
    host_name = socket.gethostname()
    return host_name + "-group"

# --------------------------------------------------------------------------------------------------------------------

if __name__ == '__main__':

    # ----------------------------------------------------------------------------------------------------------------
    # cmd...

    cmd = CmdAWSGroupSetup()

    if not cmd.is_valid():
        cmd.print_help(sys.stderr)
        exit(2)

    if cmd.verbose:
        print("aws_group_setup: %s" % cmd, file=sys.stderr)
        sys.stderr.flush()

    # ----------------------------------------------------------------------------------------------------------------
    # resources...

    aws_group_name = cmd.aws_group_name if cmd.aws_group_name else return_group_name()
    aws_region = "us-west-2"

    # ----------------------------------------------------------------------------------------------------------------
    # run...

    # ClientAuth...
    awsGroupConf = AWSGroupConfigurator.load(Host)

    if cmd.set:
        if awsGroupConf:
            user_choice = input("Group configuration already exists. Type Yes to update: ")
            print("")
            if not user_choice.lower() == "yes":
                print("Operation cancelled")
                exit()
        try:
            aws_configurator = AWSGroupConfigurator(aws_group_name, create_aws_client(), cmd.use_ml)

            aws_configurator.collect_information(Host)
            aws_configurator.define_aws_group_resources(Host)
            aws_configurator.define_aws_group_functions()
            aws_configurator.define_aws_group_subscriptions()
            aws_configurator.create_aws_group_definition()
            aws_configurator.save(Host)

        except ClientError as error:
            if error.response['Error']['Code'] == 'BadRequestException':
                print("aws_group_setup: Invalid request.", file=sys.stderr)
            if error.response['Error']['Code'] == 'InternalServerErrorException':
                print("aws_group_setup: AWS server error.", file=sys.stderr)

        except ProjectMissingError:
            print("aws_group_setup: Project configuration not set.", file=sys.stderr)

    if cmd.show_current:

        try:
            aws_group_info = AWSGroup(cmd.aws_group_name, create_aws_client())

            aws_group_info.get_group_info_from_name()
            aws_group_info.get_group_arns()
            aws_group_info.output_current_info()

            print(JSONify.dumps(aws_group_info))
        except KeyError:
            print("Group may not of been configured", file=sys.stderr)

<|endoftext|>


<commit_before>"""
:copyright: 2010-2015 by Ronny Pfannschmidt
:license: MIT
"""
from __future__ import annotations

import os
import warnings
from typing import Any
from typing import Callable
from typing import TYPE_CHECKING

from ._entrypoints import _call_entrypoint_fn
from ._entrypoints import _version_from_entrypoints
from ._overrides import _read_pretended_version_for
from ._overrides import PRETEND_KEY
from ._overrides import PRETEND_KEY_NAMED
from ._version_cls import _version_as_tuple
from ._version_cls import NonNormalizedVersion
from ._version_cls import Version
from .config import Configuration
from .config import DEFAULT_LOCAL_SCHEME
from .config import DEFAULT_TAG_REGEX
from .config import DEFAULT_VERSION_SCHEME
from .discover import iter_matching_entrypoints
from .utils import function_has_arg
from .utils import trace
from .version import format_version
from .version import meta
from .version import ScmVersion

if TYPE_CHECKING:
    from typing import NoReturn

    from . import _types as _t

TEMPLATES = {
    ".py": """\
# coding: utf-8
# file generated by setuptools_scm
# don't change, don't track in version control
__version__ = version = {version!r}
__version_tuple__ = version_tuple = {version_tuple!r}
""",
    ".txt": "{version}",
}

def version_from_scm(root: _t.PathT) -> ScmVersion | None:
    warnings.warn(
        "version_from_scm is deprecated please use get_version",
        category=DeprecationWarning,
        stacklevel=2,
    )
    config = Configuration(root=root)
    return _version_from_entrypoints(config)

def dump_version(
    root: _t.PathT,
    version: str,
    write_to: _t.PathT,
    template: str | None = None,
) -> None:
    assert isinstance(version, str)
    target = os.path.normpath(os.path.join(root, write_to))
    ext = os.path.splitext(target)[1]
    template = template or TEMPLATES.get(ext)

    if template is None:
        raise ValueError(
            "bad file format: '{}' (of {}) \nonly *.txt and *.py are supported".format(
                os.path.splitext(target)[1], target
            )
        )
    version_tuple = _version_as_tuple(version)

    with open(target, "w") as fp:
        fp.write(template.format(version=version, version_tuple=version_tuple))

def _do_parse(config: Configuration) -> ScmVersion | None:
    pretended = _read_pretended_version_for(config)
    if pretended is not None:
        return pretended

    if config.parse:
        parse_result = _call_entrypoint_fn(config.absolute_root, config, config.parse)
        if isinstance(parse_result, str):
            raise TypeError(
                f"version parse result was {str!r}\nplease return a parsed version"
            )
        version: ScmVersion | None
        if parse_result:
            assert isinstance(parse_result, ScmVersion)
            version = parse_result
        else:
            version = _version_from_entrypoints(config, fallback=True)
    else:
        # include fallbacks after dropping them from the main entrypoint
        version = _version_from_entrypoints(config) or _version_from_entrypoints(
            config, fallback=True
        )

    return version

def _version_missing(config: Configuration) -> NoReturn:
    raise LookupError(
        f"setuptools-scm was unable to detect version for {config.absolute_root}.\n\n"
        "Make sure you're either building from a fully intact git repository "
        "or PyPI tarballs. Most other sources (such as GitHub's tarballs, a "
        "git checkout without the .git folder) don't contain the necessary "
        "metadata and will not work.\n\n"
        "For example, if you're using pip, instead of "
        "https://github.com/user/proj/archive/master.zip "
        "use git+https://github.com/user/proj.git#egg=proj"
    )

def get_version(
    root: str = ".",
    version_scheme: Callable[[ScmVersion], str] | str = DEFAULT_VERSION_SCHEME,
    local_scheme: Callable[[ScmVersion], str] | str = DEFAULT_LOCAL_SCHEME,
    write_to: _t.PathT | None = None,
    write_to_template: str | None = None,
    relative_to: str | None = None,
    tag_regex: str = DEFAULT_TAG_REGEX,
    parentdir_prefix_version: str | None = None,
    fallback_version: str | None = None,
    fallback_root: _t.PathT = ".",
    parse: Any | None = None,
    git_describe_command: Any | None = None,
    dist_name: str | None = None,
    version_cls: Any | None = None,
    normalize: bool = True,
    search_parent_directories: bool = False,
) -> str:
    """
    If supplied, relative_to should be a file from which root may
    be resolved. Typically called by a script or module that is not
    in the root of the repository to direct setuptools_scm to the
    root of the repository by supplying ``__file__``.
    """

    config = Configuration(**locals())
    maybe_version = _get_version(config)
    if maybe_version is None:
        _version_missing(config)
    return maybe_version

def _get_version(config: Configuration) -> str | None:
    parsed_version = _do_parse(config)
    if parsed_version is None:
        return None
    version_string = format_version(
        parsed_version,
        version_scheme=config.version_scheme,
        local_scheme=config.local_scheme,
    )
    if config.write_to is not None:
        dump_version(
            root=config.root,
            version=version_string,
            write_to=config.write_to,
            template=config.write_to_template,
        )

    return version_string

# Public API
__all__ = [
    "get_version",
    "dump_version",
    "version_from_scm",
    "Configuration",
    "DEFAULT_VERSION_SCHEME",
    "DEFAULT_LOCAL_SCHEME",
    "DEFAULT_TAG_REGEX",
    "PRETEND_KEY",
    "PRETEND_KEY_NAMED",
    "Version",
    "NonNormalizedVersion",
    # TODO: are the symbols below part of public API ?
    "function_has_arg",
    "trace",
    "format_version",
    "meta",
    "iter_matching_entrypoints",
]
<commit_msg>Remove default "coding: utf-8" in .py template<commit_after>"""
:copyright: 2010-2015 by Ronny Pfannschmidt
:license: MIT
"""
from __future__ import annotations

import os
import warnings
from typing import Any
from typing import Callable
from typing import TYPE_CHECKING

from ._entrypoints import _call_entrypoint_fn
from ._entrypoints import _version_from_entrypoints
from ._overrides import _read_pretended_version_for
from ._overrides import PRETEND_KEY
from ._overrides import PRETEND_KEY_NAMED
from ._version_cls import _version_as_tuple
from ._version_cls import NonNormalizedVersion
from ._version_cls import Version
from .config import Configuration
from .config import DEFAULT_LOCAL_SCHEME
from .config import DEFAULT_TAG_REGEX
from .config import DEFAULT_VERSION_SCHEME
from .discover import iter_matching_entrypoints
from .utils import function_has_arg
from .utils import trace
from .version import format_version
from .version import meta
from .version import ScmVersion

if TYPE_CHECKING:
    from typing import NoReturn

    from . import _types as _t

TEMPLATES = {
    ".py": """\
# file generated by setuptools_scm
# don't change, don't track in version control
__version__ = version = {version!r}
__version_tuple__ = version_tuple = {version_tuple!r}
""",
    ".txt": "{version}",
}

def version_from_scm(root: _t.PathT) -> ScmVersion | None:
    warnings.warn(
        "version_from_scm is deprecated please use get_version",
        category=DeprecationWarning,
        stacklevel=2,
    )
    config = Configuration(root=root)
    return _version_from_entrypoints(config)

def dump_version(
    root: _t.PathT,
    version: str,
    write_to: _t.PathT,
    template: str | None = None,
) -> None:
    assert isinstance(version, str)
    target = os.path.normpath(os.path.join(root, write_to))
    ext = os.path.splitext(target)[1]
    template = template or TEMPLATES.get(ext)

    if template is None:
        raise ValueError(
            "bad file format: '{}' (of {}) \nonly *.txt and *.py are supported".format(
                os.path.splitext(target)[1], target
            )
        )
    version_tuple = _version_as_tuple(version)

    with open(target, "w") as fp:
        fp.write(template.format(version=version, version_tuple=version_tuple))

def _do_parse(config: Configuration) -> ScmVersion | None:
    pretended = _read_pretended_version_for(config)
    if pretended is not None:
        return pretended

    if config.parse:
        parse_result = _call_entrypoint_fn(config.absolute_root, config, config.parse)
        if isinstance(parse_result, str):
            raise TypeError(
                f"version parse result was {str!r}\nplease return a parsed version"
            )
        version: ScmVersion | None
        if parse_result:
            assert isinstance(parse_result, ScmVersion)
            version = parse_result
        else:
            version = _version_from_entrypoints(config, fallback=True)
    else:
        # include fallbacks after dropping them from the main entrypoint
        version = _version_from_entrypoints(config) or _version_from_entrypoints(
            config, fallback=True
        )

    return version

def _version_missing(config: Configuration) -> NoReturn:
    raise LookupError(
        f"setuptools-scm was unable to detect version for {config.absolute_root}.\n\n"
        "Make sure you're either building from a fully intact git repository "
        "or PyPI tarballs. Most other sources (such as GitHub's tarballs, a "
        "git checkout without the .git folder) don't contain the necessary "
        "metadata and will not work.\n\n"
        "For example, if you're using pip, instead of "
        "https://github.com/user/proj/archive/master.zip "
        "use git+https://github.com/user/proj.git#egg=proj"
    )

def get_version(
    root: str = ".",
    version_scheme: Callable[[ScmVersion], str] | str = DEFAULT_VERSION_SCHEME,
    local_scheme: Callable[[ScmVersion], str] | str = DEFAULT_LOCAL_SCHEME,
    write_to: _t.PathT | None = None,
    write_to_template: str | None = None,
    relative_to: str | None = None,
    tag_regex: str = DEFAULT_TAG_REGEX,
    parentdir_prefix_version: str | None = None,
    fallback_version: str | None = None,
    fallback_root: _t.PathT = ".",
    parse: Any | None = None,
    git_describe_command: Any | None = None,
    dist_name: str | None = None,
    version_cls: Any | None = None,
    normalize: bool = True,
    search_parent_directories: bool = False,
) -> str:
    """
    If supplied, relative_to should be a file from which root may
    be resolved. Typically called by a script or module that is not
    in the root of the repository to direct setuptools_scm to the
    root of the repository by supplying ``__file__``.
    """

    config = Configuration(**locals())
    maybe_version = _get_version(config)
    if maybe_version is None:
        _version_missing(config)
    return maybe_version

def _get_version(config: Configuration) -> str | None:
    parsed_version = _do_parse(config)
    if parsed_version is None:
        return None
    version_string = format_version(
        parsed_version,
        version_scheme=config.version_scheme,
        local_scheme=config.local_scheme,
    )
    if config.write_to is not None:
        dump_version(
            root=config.root,
            version=version_string,
            write_to=config.write_to,
            template=config.write_to_template,
        )

    return version_string

# Public API
__all__ = [
    "get_version",
    "dump_version",
    "version_from_scm",
    "Configuration",
    "DEFAULT_VERSION_SCHEME",
    "DEFAULT_LOCAL_SCHEME",
    "DEFAULT_TAG_REGEX",
    "PRETEND_KEY",
    "PRETEND_KEY_NAMED",
    "Version",
    "NonNormalizedVersion",
    # TODO: are the symbols below part of public API ?
    "function_has_arg",
    "trace",
    "format_version",
    "meta",
    "iter_matching_entrypoints",
]
<|endoftext|>


<commit_before>"""Results provides a common interface to access results from model runs.

Raises
------
SmifDataNotFoundError
    If data cannot be found in the store when try to read from the store
SmifDataMismatchError
    Data presented to read, write and update methods is in the
    incorrect format or of wrong dimensions to that expected
SmifDataReadError
    When unable to read data e.g. unable to handle file type or connect
    to database
"""

import os

from smif.data_layer.file import (CSVDataStore, FileMetadataStore,
                                  ParquetDataStore, YamlConfigStore)
from smif.data_layer.store import Store

class Results:
    """Common interface to access results from model runs.

    Parameters
    ----------
    details_dict: dict optional dictionary of the form {'interface': <interface>, 'dir': <dir>}
        where <interface> is either 'local_csv' or 'local_parquet', and <dir> is the model base
        directory
    store: Store optional pre-created Store object
    """
    def __init__(self, details_dict: dict = None, store: Store = None):

        assert bool(details_dict) != bool(store),\
            'Results() accepts either a details dict or a store'

        self._store = store
        if store:
            return

        try:
            interface = details_dict['interface']
        except KeyError:
            print('No interface provided for Results().  Assuming local_csv.')
            interface = 'local_csv'

        try:
            directory = details_dict['dir']
        except KeyError:
            print('No directory provided for Results().  Assuming \'.\'.')
            directory = '.'

        # Check that the provided interface is supported
        file_store = self._get_file_store(interface)
        if file_store is None:
            raise ValueError(
                'Unsupported interface "{}". Supply local_csv or local_parquet'.format(
                    interface))

        # Check that the directory is valid
        if not os.path.isdir(directory):
            raise ValueError('Expected {} to be a valid directory'.format(directory))

        self._store = Store(
            config_store=YamlConfigStore(directory),
            metadata_store=FileMetadataStore(directory),
            data_store=file_store(directory),
            model_base_folder=directory
        )

    @staticmethod
    def _get_file_store(interface):
        """ Return the appropriate derived FileDataStore class, or None if the requested
        interface is invalid.

        Parameters
        ----------
        interface: str the requested interface

        Returns
        -------
        The appropriate derived FileDataStore class
        """
        return {
            'local_csv': CSVDataStore,
            'local_parquet': ParquetDataStore,
        }.get(interface, None)

    def list_model_runs(self):
        """ Return a list of model run names.

        Returns
        -------
        List of model run names
        """
        return sorted([x['name'] for x in self._store.read_model_runs()])

    def available_results(self, model_run_name):
        """ Return the results available for a given model run.

        Parameters
        ----------
        model_run_name: str the requested model run

        Returns
        -------
        A nested dictionary data structure of the results available for the given model run
        """

        available = self._store.available_results(model_run_name)

        results = {
            'model_run': model_run_name,
            'sos_model': self._store.read_model_run(model_run_name)['sos_model'],
            'sector_models': dict(),
        }

        model_names = {sec for _t, _d, sec, _out in available}
        for model_name in model_names:
            results['sector_models'][model_name] = {
                'outputs': dict(),
            }

            outputs = {out for _t, _d, sec, out in available if sec == model_name}

            for output in outputs:
                results['sector_models'][model_name]['outputs'][output] = dict()

                decs = {d for _t, d, sec, out in available if
                        sec == model_name and out == output}

                for dec in decs:
                    ts = sorted({t for t, d, sec, out in available if
                                 d == dec and sec == model_name and out == output})
                    results['sector_models'][model_name]['outputs'][output][dec] = ts

        return results

    def read(self,
             model_run_names: list,
             sec_model_names: list,
             output_names: list,
             timesteps: list = None,
             decisions: list = None,
             time_decision_tuples: list = None,
             ):
        """ Return the results from the store.

        Parameters
        ----------
        model_run_names: list the requested model run names
        sec_model_names: list the requested sector model names (exactly one required)
        output_names: list the requested output names (exactly one required)
        timesteps: list the requested timesteps
        decisions: list the requested decision iterations
        time_decision_tuples: list a list of requested (timestep, decision) tuples

        Returns
        -------
        A dictionary of DataArrays, a single DataArray for each model run
        """

        if len(sec_model_names) != 1:
            raise NotImplementedError(
                'Results.read() currently requires exactly one sector model'
            )

        if len(output_names) != 1:
            raise NotImplementedError(
                'Results.read() currently requires exactly one output'
            )

        return self._store.get_results(
            model_run_names,
            sec_model_names[0],
            output_names[0],
            timesteps,
            decisions,
            time_decision_tuples
        )
<commit_msg>#359 Return a dataframe with cols for model run, timestep and decision<commit_after>"""Results provides a common interface to access results from model runs.

Raises
------
SmifDataNotFoundError
    If data cannot be found in the store when try to read from the store
SmifDataMismatchError
    Data presented to read, write and update methods is in the
    incorrect format or of wrong dimensions to that expected
SmifDataReadError
    When unable to read data e.g. unable to handle file type or connect
    to database
"""

import os

import pandas as pd
from smif.data_layer.file import (CSVDataStore, FileMetadataStore,
                                  ParquetDataStore, YamlConfigStore)
from smif.data_layer.store import Store

class Results:
    """Common interface to access results from model runs.

    Parameters
    ----------
    details_dict: dict optional dictionary of the form {'interface': <interface>, 'dir': <dir>}
        where <interface> is either 'local_csv' or 'local_parquet', and <dir> is the model base
        directory
    store: Store optional pre-created Store object
    """

    def __init__(self, details_dict: dict = None, store: Store = None):

        assert bool(details_dict) != bool(store), \
            'Results() accepts either a details dict or a store'

        self._store = store
        if store:
            return

        try:
            interface = details_dict['interface']
        except KeyError:
            print('No interface provided for Results().  Assuming local_csv.')
            interface = 'local_csv'

        try:
            directory = details_dict['dir']
        except KeyError:
            print('No directory provided for Results().  Assuming \'.\'.')
            directory = '.'

        # Check that the provided interface is supported
        file_store = self._get_file_store(interface)
        if file_store is None:
            raise ValueError(
                'Unsupported interface "{}". Supply local_csv or local_parquet'.format(
                    interface))

        # Check that the directory is valid
        if not os.path.isdir(directory):
            raise ValueError('Expected {} to be a valid directory'.format(directory))

        self._store = Store(
            config_store=YamlConfigStore(directory),
            metadata_store=FileMetadataStore(directory),
            data_store=file_store(directory),
            model_base_folder=directory
        )

    @staticmethod
    def _get_file_store(interface):
        """ Return the appropriate derived FileDataStore class, or None if the requested
        interface is invalid.

        Parameters
        ----------
        interface: str the requested interface

        Returns
        -------
        The appropriate derived FileDataStore class
        """
        return {
            'local_csv': CSVDataStore,
            'local_parquet': ParquetDataStore,
        }.get(interface, None)

    def list_model_runs(self):
        """ Return a list of model run names.

        Returns
        -------
        List of model run names
        """
        return sorted([x['name'] for x in self._store.read_model_runs()])

    def available_results(self, model_run_name):
        """ Return the results available for a given model run.

        Parameters
        ----------
        model_run_name: str the requested model run

        Returns
        -------
        A nested dictionary data structure of the results available for the given model run
        """

        available = self._store.available_results(model_run_name)

        results = {
            'model_run': model_run_name,
            'sos_model': self._store.read_model_run(model_run_name)['sos_model'],
            'sector_models': dict(),
        }

        model_names = {sec for _t, _d, sec, _out in available}
        for model_name in model_names:
            results['sector_models'][model_name] = {
                'outputs': dict(),
            }

            outputs = {out for _t, _d, sec, out in available if sec == model_name}

            for output in outputs:
                results['sector_models'][model_name]['outputs'][output] = dict()

                decs = {d for _t, d, sec, out in available if
                        sec == model_name and out == output}

                for dec in decs:
                    ts = sorted({t for t, d, sec, out in available if
                                 d == dec and sec == model_name and out == output})
                    results['sector_models'][model_name]['outputs'][output][dec] = ts

        return results

    def read(self,
             model_run_names: list,
             sec_model_names: list,
             output_names: list,
             timesteps: list = None,
             decisions: list = None,
             time_decision_tuples: list = None,
             ):
        """ Return the results from the store.

        Parameters
        ----------
        model_run_names: list the requested model run names
        sec_model_names: list the requested sector model names (exactly one required)
        output_names: list the requested output names (exactly one required)
        timesteps: list the requested timesteps
        decisions: list the requested decision iterations
        time_decision_tuples: list a list of requested (timestep, decision) tuples

        Returns
        -------
        A dictionary of DataArrays, a single DataArray for each model run
        """

        if len(sec_model_names) != 1:
            raise NotImplementedError(
                'Results.read() currently requires exactly one sector model'
            )

        if len(output_names) != 1:
            raise NotImplementedError(
                'Results.read() currently requires exactly one output'
            )

        results_dict = self._store.get_results(
            model_run_names,
            sec_model_names[0],
            output_names[0],
            timesteps,
            decisions,
            time_decision_tuples
        )

        # Get each DataArray as a pandas data frame and concatenate, resetting the index to
        # give back a flat data array
        list_of_df = [x.as_df() for x in results_dict.values()]
        names_of_df = [x for x in results_dict.keys()]

        results = pd.concat(list_of_df, keys=names_of_df, names=['model_run']).reset_index()

        # Unpack the timestep_decision tuples into individual columns and return
        results[['timestep', 'decision']] = pd.DataFrame(results['timestep_decision'].tolist(),
                                                         index=results.index)

        return results.drop(columns=['timestep_decision'])
<|endoftext|>


<commit_before>
import re

from streamlink.plugin import Plugin
from streamlink.plugin.api import useragents, validate
from streamlink.stream import HLSStream
from streamlink.utils import parse_json, update_scheme

class IDF1(Plugin):
    DACAST_API_URL = 'https://json.dacast.com/b/{}/{}/{}'
    DACAST_TOKEN_URL = 'https://services.dacast.com/token/i/b/{}/{}/{}'

    _url_re = re.compile(r'http://www\.idf1\.fr/(videos/[^/]+/[^/]+\.html|live\b)')
    _video_id_re = re.compile(r"dacast\('(?P<broadcaster_id>\d+)_(?P<video_type>[a-z]+)_(?P<video_id>\d+)', 'replay_content', data\);")
    _video_id_alt_re = re.compile(r'<script src="//player.dacast.com/js/player.js" id="(?P<broadcaster_id>\d+)_(?P<video_type>[cf])_(?P<video_id>\d+)"')
    _player_url = 'http://ssl.p.jwpcdn.com/player/v/7.12.6/jwplayer.flash.swf'

    _api_schema = validate.Schema(
        validate.transform(parse_json),
        {
            validate.optional('html5'): validate.all(
                [
                    {
                        'src': validate.url()
                    },
                ],
            ),
            'hls': validate.url(),
            'hds': validate.url()
        },
        validate.transform(lambda x: [update_scheme(IDF1.DACAST_API_URL, x['hls']), x['hds']] + [y['src'] for y in x.get('html5', [])])
    )

    _token_schema = validate.Schema(
        validate.transform(parse_json),
        {'token': validate.text},
        validate.get('token')
    )

    _user_agent = useragents.IE_11

    @classmethod
    def can_handle_url(cls, url):
        return IDF1._url_re.match(url)

    def _get_streams(self):
        res = self.session.http.get(self.url)
        match = self._video_id_re.search(res.text) or self._video_id_alt_re.search(res.text)
        if match is None:
            return
        broadcaster_id = match.group('broadcaster_id')
        video_type = match.group('video_type')
        video_id = match.group('video_id')

        videos = self.session.http.get(self.DACAST_API_URL.format(broadcaster_id, video_type, video_id), schema=self._api_schema)
        token = self.session.http.get(self.DACAST_TOKEN_URL.format(broadcaster_id, video_type, video_id), schema=self._token_schema)
        parsed = []

        for video_url in videos:
            video_url += token

            # Ignore duplicate video URLs
            if video_url in parsed:
                continue
            parsed.append(video_url)

            # Ignore HDS streams (broken)
            if '.m3u8' in video_url:
                for s in HLSStream.parse_variant_playlist(self.session, video_url).items():
                    yield s

__plugin__ = IDF1
<commit_msg>[plugins.idf1] Update for new website layout<commit_after>
import re

from streamlink.plugin import Plugin
from streamlink.plugin.api import useragents, validate
from streamlink.stream import HLSStream
from streamlink.utils import parse_json, update_scheme

class IDF1(Plugin):
    DACAST_API_URL = 'https://json.dacast.com/b/{}/{}/{}'
    DACAST_TOKEN_URL = 'https://services.dacast.com/token/i/b/{}/{}/{}'

    _url_re = re.compile(r'http://www\.idf1\.fr/(videos/[^/]+/[^/]+\.html|live\b)')
    _video_id_re = re.compile(r"dacast\('(?P<broadcaster_id>\d+)_(?P<video_type>[a-z]+)_(?P<video_id>\d+)', 'replay_content', data\);")
    _video_id_alt_re = re.compile(r'<script src="https://player.dacast.com/js/player.js" id="(?P<broadcaster_id>\d+)_(?P<video_type>[cf])_(?P<video_id>\d+)"')
    _player_url = 'http://ssl.p.jwpcdn.com/player/v/7.12.6/jwplayer.flash.swf'

    _api_schema = validate.Schema(
        validate.transform(parse_json),
        {
            validate.optional('html5'): validate.all(
                [
                    {
                        'src': validate.url()
                    },
                ],
            ),
            'hls': validate.url(),
            'hds': validate.url()
        },
        validate.transform(lambda x: [update_scheme(IDF1.DACAST_API_URL, x['hls']), x['hds']] + [y['src'] for y in x.get('html5', [])])
    )

    _token_schema = validate.Schema(
        validate.transform(parse_json),
        {'token': validate.text},
        validate.get('token')
    )

    _user_agent = useragents.IE_11

    @classmethod
    def can_handle_url(cls, url):
        return IDF1._url_re.match(url)

    def _get_streams(self):
        res = self.session.http.get(self.url)
        match = self._video_id_re.search(res.text) or self._video_id_alt_re.search(res.text)
        if match is None:
            return
        broadcaster_id = match.group('broadcaster_id')
        video_type = match.group('video_type')
        video_id = match.group('video_id')

        videos = self.session.http.get(self.DACAST_API_URL.format(broadcaster_id, video_type, video_id), schema=self._api_schema)
        token = self.session.http.get(
            self.DACAST_TOKEN_URL.format(broadcaster_id, video_type, video_id),
            schema=self._token_schema,
            headers={'referer': self.url}
        )
        parsed = []

        for video_url in videos:
            video_url += token

            # Ignore duplicate video URLs
            if video_url in parsed:
                continue
            parsed.append(video_url)

            # Ignore HDS streams (broken)
            if '.m3u8' in video_url:
                for s in HLSStream.parse_variant_playlist(self.session, video_url).items():
                    yield s

__plugin__ = IDF1
<|endoftext|>


<commit_before>__author__ = 's7a'

# All imports
from nltk.tree import Tree

# The Relative clauses class
class RelativeClauses:

    # Constructor for the Relative Clauses class
    def __init__(self):
        self.has_wh_word = False

    # Break the tree
    def break_tree(self, tree):
        t = Tree.fromstring(str(tree))

        self.has_wh_word = False
        self.parse_tree(t)

        print "Relative Clause: " + str(self.has_wh_word)

        result_string = ""
        return result_string

    # Parse the tree
    def parse_tree(self, tree):
        if type(tree) == Tree:
            self.has_wh_word |= tree.label() == "WHNP"
            for node in tree:
                self.parse_tree(node)<commit_msg>Fix detection of relative clause<commit_after>__author__ = 's7a'

# All imports
from nltk.tree import Tree

# The Relative clauses class
class RelativeClauses:

    # Constructor for the Relative Clauses class
    def __init__(self):
        self.has_wh_word = False

    # Break the tree
    def break_tree(self, tree):
        t = Tree.fromstring(str(tree))

        self.has_wh_word = False
        self.parse_tree(t)

        print "Relative Clause: " + str(self.has_wh_word)

        result_string = ""
        return result_string

    # Parse the tree
    def parse_tree(self, tree):

        if type(tree) == Tree:
            if tree.label() == "SBAR":
                for node in tree:
                    if type(node) == Tree:
                        self.has_wh_word |= node.label() == "WHNP"

            for node in tree:
                self.parse_tree(node)<|endoftext|>