<commit_before>from datetime import datetime
from docset import build_docset
from docset.rust import templates
from jinja2 import Template
from invoke import run, task
import importlib
import os
import requests
import shutil
import sys
import tarfile
from tempfile import TemporaryFile, mkdtemp
import toml

NIGHTLY_URL = "http://static.rust-lang.org/dist/rust-nightly-x86_64-unknown-linux-gnu.tar.gz"
TAG_FILE = ".last-tag"
DOC_PREFIX = "rust-nightly-x86_64-unknown-linux-gnu/doc"

def tag_from_response(req):
    """Generates a uniq tag from a response"""
    return req.headers['etag']

def has_nightly_update():
    """Checks if tag was changed"""
    if not os.path.exists(TAG_FILE):
        return True
    else:
        with open(TAG_FILE, "rt") as f:
            last_tag = f.read().strip()

    r = requests.head(NIGHTLY_URL)
    new_tag = tag_from_response(r)
    return new_tag != last_tag

def extract_docs(tar, dest_dir):
    """Extracts only documentation files from nightly archive"""
    for info in tar.getmembers():
        if info.name.startswith(DOC_PREFIX):
            rel_path = os.path.relpath(info.name, DOC_PREFIX)
            dest_path = os.path.join(dest_dir, rel_path)
            if info.isdir():
                if not os.path.exists(dest_path):
                    os.makedirs(dest_path)
            else:
                in_file = tar.extractfile(info)
                with open(dest_path, "wb") as out_file:
                    out_file.write(in_file.read())

@task
def update_nightly(force=False, out_dir="nightly_out"):
    """Checks for update and re-builds doc if required. Stores new tag"""
    if not force:
        if not has_nightly_update():
            print "No updates available yet"
            return

    r = requests.get(NIGHTLY_URL, stream=True)
    with TemporaryFile("w+b") as temp_file:
        print "Downloading"
        for chunk in r.iter_content(1024*1024*10):
            temp_file.write(chunk)

        temp_dir = mkdtemp()
        try:
            print "Extracting to", temp_dir
            temp_file.seek(0)
            with tarfile.open(fileobj=temp_file, mode='r:gz',
                              bufsize=1024*1024*10) as tar:
                extract_docs(tar, temp_dir)

            print "Building docset"
            build(doc_dir=temp_dir, out_dir=out_dir, conf="nightly.toml")

            with open(TAG_FILE, "w+t") as f:
                f.write(tag_from_response(r))
        except:
            import traceback
            traceback.print_exc()
            shutil.rmtree(temp_dir)

def mod_with_name(name, error_fmt):
    try:
        return importlib.import_module(name)
    except ImportError as e:
        print error_fmt % {'name': name}
        sys.exit(2)

def ensure_abs(path, prefix):
    if os.path.isabs(path):
        return path
    else:
        return os.path.join(prefix, path)

def not_empty(key):
    def __closure__(section, data):
        if (key not in data) or (not data[key]):
            return "[%s.%s] should have a value" % (section, key,)
        return None

    return __closure__

def flatten(lst_of_lst):
    def concat(a, b):
        a.extend(b)
        return a
    return reduce(lambda a, b: concat(a, b), lst_of_lst, [])

def validate_section(cfg, section_name, rules):
    if not section_name in cfg:
        return ["Section [%s] must be presented" % section_name]
    else:
        data = cfg[section_name]
        return filter(lambda x: x != None,
                      map(lambda r: r(section_name, data), rules))

def validate_config(cfg):
    sections = {
        'docset': [not_empty('name'), not_empty('type'), not_empty('bundle_id')],
    }

    # feed is optional section
    if 'feed' in cfg:
        sections['feed'] = [not_empty('base_url')]

    return flatten(map(lambda (s, r): validate_section(cfg, s, r),
                       sections.items()))

def template_from(info, key, prefix_path, default):
    if not key in info:
        data = default
    else:
        path = ensure_abs(info[key], prefix_path)
        with open(template_path, "rt") as f:
            data = f.read()
    return Template(data)

def exit_with(code, message):
    if isinstance(message, str):
        sys.stdout.write(message)
        sys.stdout.write("\n")
    elif isinstance(message, list):
        for l in message:
            sys.stdout.write(l)
            sys.stdout.write("\n")
    sys.stdout.flush()
    sys.exit(code)

def warn(message):
    sys.stdout.write(message)
    sys.stdout.write("\n")
    sys.stdout.flush()

@task
def build(doc_dir=None, out_dir=None, conf="docset.toml"):
    """Builds a docset from doc_dir using settings.
Command line arguments to doc_dir and out_dir override
corresponding conf values.

Warning: out dir is cleaned before"""
    conf = os.path.abspath(conf)

    if not os.path.exists(conf):
        exit_with(1, "There is no configuration at path %s" % conf)

    with open(conf) as cf:
        config = toml.loads(cf.read())

    errors = validate_config(config)
    if errors:
        exit_with(1, errors)

    ds = config['docset']
    if not 'version' in ds:
        def_version = "0.1"
        warn("No [docset.version] found, defaulting to %s" % def_version)
        ds['version'] = def_version

    plist_template = template_from(ds, 'plist', os.path.dirname(conf), templates.INFO_PLIST)
    ds['plist'] = plist_template.render({
        'time': datetime.now(),
        'name': ds['name'],
        'version': ds['version'],
        'info': ds,
        'bundle_id': ds['bundle_id']
    })

    if 'icon' in ds:
        ds['icon'] = ensure_abs(ds['icon'], os.path.dirname(conf))

    parts = ds['type'].split(":")
    if len(parts) == 1:
        parts.append("default")

    ty_mod = mod_with_name(parts[0], "Failed to import docset type specs: %(name)s")
    rules = ty_mod.__getattribute__(parts[1])

    warns = {}
    if not doc_dir:
        if 'doc_dir' in ds:
            doc_dir = ensure_abs(ds['doc_dir'], os.path.dirname(conf))
        else:
            warns['doc_dir'] = True
            doc_dir = '.'

    if not out_dir:
        if 'out_dir' in ds:
            out_dir = ensure_abs(ds['out_dir'], os.path.dirname(conf))
        else:
            warns['out_dir'] = True
            out_dir = 'ds_out'

    doc_dir = os.path.abspath(doc_dir)
    out_dir = os.path.abspath(out_dir)

    # Have to delay to get full path
    if 'doc_dir' in warns:
        warn("doc_dir wasn't specified, defaulting to %s" % doc_dir)

    if 'out_dir' in warns:
        warn("out_dir wasn't specified, defaulting to %s" % out_dir)

    if os.path.exists(out_dir):
        if os.path.isdir(out_dir):
            shutil.rmtree(out_dir)
        else:
            exit_with(1, "Output dir points to a file!\n")

    os.makedirs(out_dir)

    build_docset(ds, rules, doc_dir, out_dir)

    if 'feed' in config:
        feed = config['feed']
        template = template_from(ds, 'template', os.path.dirname(conf), templates.FEED_XML)

        TGZ_TEMP = os.path.join(out_dir, "%s.tar.gz" % ds['name'])
        DOC_DIR = os.path.join(out_dir, "%s.docset" % ds['name'])
        run("tar --exclude='.DS_Store'  -s '#%s/##s' --options gzip:9 -czf %s %s" % (out_dir, TGZ_TEMP, DOC_DIR))

        sha = run('openssl sha1 %s | cut -d "=" -f 2' % TGZ_TEMP, hide="out").stdout.strip()
        TGZ_NAME = "%s-%s.tgz" % (ds['name'], sha[:8],)
        TGZ = os.path.join(out_dir, TGZ_NAME)

        xml = template.render({
            'sha': sha,
            'time': datetime.now(),
            'name': ds['name'],
            'file_name': TGZ_NAME,
            'version': ds['version'],
            'info': ds,
            'base_url': feed['base_url']
        })
        feed_xml_path = os.path.join(out_dir, "%s.xml" % ds['name'])

        with open(feed_xml_path, "w+t") as f:
            f.write(xml)

        run("mv %s %s" % (TGZ_TEMP, TGZ,))

        if 'upload_cmd' in feed:
            run("%s %s %s" % (feed['upload_cmd'], TGZ, feed_xml_path,))
<commit_msg>Dropping shell commands in favour of Python code<commit_after>from datetime import datetime
from docset import build_docset
from docset.rust import templates
import hashlib
from invoke import run, task
import importlib
from jinja2 import Template
import os
import requests
import shutil
import sys
import tarfile
from tempfile import TemporaryFile, mkdtemp
import toml

NIGHTLY_URL = "http://static.rust-lang.org/dist/rust-nightly-x86_64-unknown-linux-gnu.tar.gz"
TAG_FILE = ".last-tag"
DOC_PREFIX = "rust-nightly-x86_64-unknown-linux-gnu/doc"

def tag_from_response(req):
    """Generates a uniq tag from a response"""
    return req.headers['etag']

def has_nightly_update():
    """Checks if tag was changed"""
    if not os.path.exists(TAG_FILE):
        return True
    else:
        with open(TAG_FILE, "rt") as f:
            last_tag = f.read().strip()

    r = requests.head(NIGHTLY_URL)
    new_tag = tag_from_response(r)
    return new_tag != last_tag

def extract_docs(tar, dest_dir):
    """Extracts only documentation files from nightly archive"""
    for info in tar.getmembers():
        if info.name.startswith(DOC_PREFIX):
            rel_path = os.path.relpath(info.name, DOC_PREFIX)
            dest_path = os.path.join(dest_dir, rel_path)
            if info.isdir():
                if not os.path.exists(dest_path):
                    os.makedirs(dest_path)
            else:
                in_file = tar.extractfile(info)
                with open(dest_path, "wb") as out_file:
                    out_file.write(in_file.read())

@task
def update_nightly(force=False, out_dir="nightly_out"):
    """Checks for update and re-builds doc if required. Stores new tag"""
    if not force:
        if not has_nightly_update():
            print "No updates available yet"
            return

    r = requests.get(NIGHTLY_URL, stream=True)
    with TemporaryFile("w+b") as temp_file:
        print "Downloading"
        for chunk in r.iter_content(1024*1024*10):
            temp_file.write(chunk)

        temp_dir = mkdtemp()
        try:
            print "Extracting to", temp_dir
            temp_file.seek(0)
            with tarfile.open(fileobj=temp_file, mode='r:gz',
                              bufsize=1024*1024*10) as tar:
                extract_docs(tar, temp_dir)

            print "Building docset"
            build(doc_dir=temp_dir, out_dir=out_dir, conf="nightly.toml")

            with open(TAG_FILE, "w+t") as f:
                f.write(tag_from_response(r))
        except:
            import traceback
            traceback.print_exc()
            shutil.rmtree(temp_dir)

def mod_with_name(name, error_fmt):
    try:
        return importlib.import_module(name)
    except ImportError as e:
        print error_fmt % {'name': name}
        sys.exit(2)

def ensure_abs(path, prefix):
    if os.path.isabs(path):
        return path
    else:
        return os.path.join(prefix, path)

def not_empty(key):
    def __closure__(section, data):
        if (key not in data) or (not data[key]):
            return "[%s.%s] should have a value" % (section, key,)
        return None

    return __closure__

def flatten(lst_of_lst):
    def concat(a, b):
        a.extend(b)
        return a
    return reduce(lambda a, b: concat(a, b), lst_of_lst, [])

def validate_section(cfg, section_name, rules):
    if not section_name in cfg:
        return ["Section [%s] must be presented" % section_name]
    else:
        data = cfg[section_name]
        return filter(lambda x: x != None,
                      map(lambda r: r(section_name, data), rules))

def validate_config(cfg):
    sections = {
        'docset': [not_empty('name'), not_empty('type'), not_empty('bundle_id')],
    }

    # feed is optional section
    if 'feed' in cfg:
        sections['feed'] = [not_empty('base_url')]

    return flatten(map(lambda (s, r): validate_section(cfg, s, r),
                       sections.items()))

def template_from(info, key, prefix_path, default):
    if not key in info:
        data = default
    else:
        path = ensure_abs(info[key], prefix_path)
        with open(template_path, "rt") as f:
            data = f.read()
    return Template(data)

def exit_with(code, message):
    if isinstance(message, str):
        sys.stdout.write(message)
        sys.stdout.write("\n")
    elif isinstance(message, list):
        for l in message:
            sys.stdout.write(l)
            sys.stdout.write("\n")
    sys.stdout.flush()
    sys.exit(code)

def warn(message):
    sys.stdout.write(message)
    sys.stdout.write("\n")
    sys.stdout.flush()

@task
def build(doc_dir=None, out_dir=None, conf="docset.toml"):
    """Builds a docset from doc_dir using settings.
Command line arguments to doc_dir and out_dir override
corresponding conf values.

Warning: out dir is cleaned before"""
    conf = os.path.abspath(conf)

    if not os.path.exists(conf):
        exit_with(1, "There is no configuration at path %s" % conf)

    with open(conf) as cf:
        config = toml.loads(cf.read())

    errors = validate_config(config)
    if errors:
        exit_with(1, errors)

    ds = config['docset']
    if not 'version' in ds:
        def_version = "0.1"
        warn("No [docset.version] found, defaulting to %s" % def_version)
        ds['version'] = def_version

    plist_template = template_from(ds, 'plist', os.path.dirname(conf), templates.INFO_PLIST)
    ds['plist'] = plist_template.render({
        'time': datetime.now(),
        'name': ds['name'],
        'version': ds['version'],
        'info': ds,
        'bundle_id': ds['bundle_id']
    })

    if 'icon' in ds:
        ds['icon'] = ensure_abs(ds['icon'], os.path.dirname(conf))

    parts = ds['type'].split(":")
    if len(parts) == 1:
        parts.append("default")

    ty_mod = mod_with_name(parts[0], "Failed to import docset type specs: %(name)s")
    rules = ty_mod.__getattribute__(parts[1])

    warns = {}
    if not doc_dir:
        if 'doc_dir' in ds:
            doc_dir = ensure_abs(ds['doc_dir'], os.path.dirname(conf))
        else:
            warns['doc_dir'] = True
            doc_dir = '.'

    if not out_dir:
        if 'out_dir' in ds:
            out_dir = ensure_abs(ds['out_dir'], os.path.dirname(conf))
        else:
            warns['out_dir'] = True
            out_dir = 'ds_out'

    doc_dir = os.path.abspath(doc_dir)
    out_dir = os.path.abspath(out_dir)

    # Have to delay to get full path
    if 'doc_dir' in warns:
        warn("doc_dir wasn't specified, defaulting to %s" % doc_dir)

    if 'out_dir' in warns:
        warn("out_dir wasn't specified, defaulting to %s" % out_dir)

    if os.path.exists(out_dir):
        if os.path.isdir(out_dir):
            shutil.rmtree(out_dir)
        else:
            exit_with(1, "Output dir points to a file!\n")

    os.makedirs(out_dir)

    build_docset(ds, rules, doc_dir, out_dir)

    if 'feed' in config:
        feed = config['feed']
        template = template_from(ds, 'template', os.path.dirname(conf), templates.FEED_XML)

        TGZ_TEMP = os.path.join(out_dir, "%s.tar.gz" % ds['name'])
        DOCSET_DIR = "%s.docset" % ds['name']
        DOC_DIR = os.path.join(out_dir, DOCSET_DIR)

        with tarfile.open(TGZ_TEMP, "w:gz", bufsize=1024*1024*10) as tar:
            tar.add(DOC_DIR, arcname = DOCSET_DIR)

        hash = hashlib.sha1()
        with open(TGZ_TEMP) as f:
            chunk = f.read(1024*1024*10)
            hash.update(chunk)

        sha = hash.hexdigest()
        TGZ_NAME = "%s-%s.tgz" % (ds['name'], sha[:8],)
        TGZ = os.path.join(out_dir, TGZ_NAME)

        xml = template.render({
            'sha': sha,
            'time': datetime.now(),
            'name': ds['name'],
            'file_name': TGZ_NAME,
            'version': ds['version'],
            'info': ds,
            'base_url': feed['base_url']
        })
        feed_xml_path = os.path.join(out_dir, "%s.xml" % ds['name'])

        with open(feed_xml_path, "w+t") as f:
            f.write(xml)

        os.rename(TGZ_TEMP, TGZ)

        if 'upload_cmd' in feed:
            run("%s %s %s" % (feed['upload_cmd'], TGZ, feed_xml_path,))
<|endoftext|>


<commit_before>#! /usr/bin/env python
"""
usage:
  tb <command> [<args>...]
  tb setup
  tb -h | --help
  tb --version

commands:
  tajima
  hmm
  filter
  call
  primer
  genome
  rename
  phylo
  freq
  geno
  vcf2tsv
  vcf2sql

"""
from docopt import docopt
from subprocess import call, check_output, CalledProcessError
from utils.vcf import *
from clint.textui import colored, puts, indent
import sys
import tb
import os

debug = None
if len(sys.argv) == 1:
    debug = [""]

def getScriptPath():
    return os.path.dirname(tb.__file__)

def main():
    args = docopt(__doc__,
                  argv=debug,
                  options_first=True)
    argv = [args['<command>']] + args['<args>']
    program_list = {"bwa": "bwa",
                    "samtools": "samtools",
                    "bcftools": "bcftools",
                    "blast": "blastn",
                    "muscle": "muscle"}
    if args["<command>"] == "setup":
        """
            Use Homebrew to install programs!
        """
        program_installed = program_list.keys()
        for install_name, program in progtbram_list.items():
            check_output(["brew", "tap", "homebrew/science"])
            try:
                with indent(4):
                    puts(colored.blue("Installing " + install_name))
                check_output(["brew", "install", install_name])
                program_installed.remove(install_name)
            except CalledProcessError:
                try:
                    check_output(["which", program])
                    with indent(4):
                        puts(colored.blue(program + " previously installed"))
                    program_installed.remove(install_name)
                except CalledProcessError:
                    with indent(4):
                        puts(colored.red("Error installing " + install_name))
        if len(program_installed) == 0:
            with indent(4):
                puts(colored.blue("Programs successfully installed!"))
        else:
            with indent(4):
                puts(colored.red("Error: Not all programs successfully installed: " + ", ".join(program_installed)))
    elif args["<command>"] == "":
        print(__doc__)
        for prog in program_list.values():
            try:
                check_output(["which", prog])
            except CalledProcessError:
                with indent(4):
                    puts(
                        colored.red(prog + " not installed. Use a package manager to install or try using 'tb.py setup'\n"))
    elif args['<command>'] in ['tajima', 'hmm', 'filter', 'call', 'primer', 'genome', 'rename', 'phylo', 'freq', "geno", "vcf2tsv", "vcf2sql", "stat"]:
        comm = ['python', getScriptPath() + '/' + args["<command>"] + ".py"] + argv
        exit(call(comm))

if __name__ == '__main__':
    main()
<commit_msg>add annotate command<commit_after>#! /usr/bin/env python
"""
usage:
  tb <command> [<args>...]
  tb setup
  tb -h | --help
  tb --version

commands:
  annotate
  tajima
  hmm
  filter
  call
  primer
  genome
  rename
  phylo
  freq
  geno
  vcf2tsv
  vcf2sql

"""
from docopt import docopt
from subprocess import call, check_output, CalledProcessError
from utils.vcf import *
from clint.textui import colored, puts, indent
import sys
import tb
import os

debug = None
if len(sys.argv) == 1:
    debug = [""]

def getScriptPath():
    return os.path.dirname(tb.__file__)

def main():
    args = docopt(__doc__,
                  argv=debug,
                  options_first=True)
    argv = [args['<command>']] + args['<args>']
    program_list = {"bwa": "bwa",
                    "samtools": "samtools",
                    "bcftools": "bcftools",
                    "blast": "blastn",
                    "muscle": "muscle"}
    if args["<command>"] == "setup":
        """
            Use Homebrew to install programs!
        """
        program_installed = program_list.keys()
        for install_name, program in progtbram_list.items():
            check_output(["brew", "tap", "homebrew/science"])
            try:
                with indent(4):
                    puts(colored.blue("Installing " + install_name))
                check_output(["brew", "install", install_name])
                program_installed.remove(install_name)
            except CalledProcessError:
                try:
                    check_output(["which", program])
                    with indent(4):
                        puts(colored.blue(program + " previously installed"))
                    program_installed.remove(install_name)
                except CalledProcessError:
                    with indent(4):
                        puts(colored.red("Error installing " + install_name))
        if len(program_installed) == 0:
            with indent(4):
                puts(colored.blue("Programs successfully installed!"))
        else:
            with indent(4):
                puts(colored.red("Error: Not all programs successfully installed: " + ", ".join(program_installed)))
    elif args["<command>"] == "":
        print(__doc__)
        for prog in program_list.values():
            try:
                check_output(["which", prog])
            except CalledProcessError:
                with indent(4):
                    puts(
                        colored.red(prog + " not installed. Use a package manager to install or try using 'tb.py setup'\n"))
    elif args['<command>'] in ['tajima', 'hmm', 'filter', 'call', 'primer', 'genome', 'rename', 'phylo', 'freq', 'geno', 'vcf2tsv', 'vcf2sql', 'stat', 'annotate']:
        comm = ['python', getScriptPath() + '/' + args["<command>"] + ".py"] + argv
        exit(call(comm))

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>"""
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements.  See the NOTICE file
distributed with this work for additional information
regarding copyright ownership.  The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License.  You may obtain a copy of the License at

http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

"""

import sys, os, pwd
from resource_management import *
from resource_management.libraries.functions import check_process_status
from subprocess import call

class Master(Script):
  def install(self, env):
    self.install_packages(env)
    import params
    import status_params

    Execute('echo Scriptdir is: ' + params.service_scriptsdir)

    Execute('echo installation directory is: ' + params.tweet_installdir)
    Execute('echo ambari host: ' + params.ambari_server_host)
    Execute('echo namenode host: ' + params.namenode_host)
    Execute('echo nimbus host: ' + params.nimbus_host)
    Execute('echo hive host: ' + params.hive_metastore_host)
    Execute('echo hbase host: ' + params.hbase_master_host)
    Execute('echo kafka broker: ' + params.kafka_broker_host)

    Directory(params.tweet_installdir, mode=0755, owner='root', group='root', recursive=True)
    Directory(status_params.tweet_piddir, mode=0755, owner='root', group='root', recursive=True)

    Execute('echo Copying script to ' + params.tweet_installdir)
    Execute('cp -f ' + params.service_scriptsdir + 'setup_nifi.sh ' + params.tweet_installdir)
    Execute('cp -f ' + params.service_scriptsdir + 'setup_twitter.sh ' + params.tweet_installdir)
    Execute('cp -f ' + params.service_scriptsdir + 'twitter_dashboard_v5.xml ' + params.tweet_installdir)
    self.configure(env)

  def configure(self, env):
    import params
    import status_params
    env.set_params(params)
    env.set_params(status_params)
    user_env=InlineTemplate(status_params.user_env)
    File(params.tweet_installdir + '/user-env.sh', content=user_env, owner='root',group='root')
    tweet_env=InlineTemplate(params.tweet_env)
    File(params.tweet_installdir + '/tweet-env.sh', content=tweet_env, owner='root',group='root')

  def stop(self, env):
    Execute ('echo stop')

  def start(self, env):
    import params
    import status_params
    self.configure(env)
    config_nifi_script = os.path.join(params.tweet_installdir,'setup_nifi.sh')
    nifi_template_xml = os.path.join(params.tweet_installdir,'twitter_dashboard_v5.xml')
    Execute ('pip install requests')
    if not os.path.exists(status_params.tweet_piddir):
        os.makedirs(status_params.tweet_piddir)
    #To change. Really bad!!!
    Execute ('echo Wait 30 seconds to let NiFi starts')
    Execute ('sleep 30')
    Execute ('chmod +x ' + config_nifi_script)
    Execute (config_nifi_script + ' ' + params.tweet_installdir + ' ' + nifi_template_xml + ' twitter_dashboard')
    config_twitter_script = os.path.join(params.tweet_installdir,'setup_twitter.sh')
    Execute ('chmod +x ' + config_twitter_script)
    Execute (config_twitter_script + ' ' + params.tweet_installdir + ' GetTwitter ')

  def status(self, env):
    import status_params
    self.check_flow_running(status_params.tweet_pidfile, "sandbox.hortonworks.com", "9090")

  def check_flow_running(self, pid_file, host, port):
    import requests
    if not pid_file or not os.path.isfile(pid_file):
      raise ComponentIsNotRunning()
    try:
      f = open(pid_file,"r")
      p_group_id = f.read()
      f.close()
      req = 'http://' + host + ':' + port + '/nifi-api/controller/process-groups/root/process-group-references/' + p_group_id.strip()
      r = requests.get(req)
      if r.status_code != requests.code.ok:
        raise ComponentIsNotRunning()
      if r.json()["processGroup"]["runningCount"] < 26:
        raise ComponentIsNotRunning()
    except Exception, e:
      raise ComponentIsNotRunning()

if __name__ == "__main__":
  Master().execute()
<commit_msg>status command<commit_after>"""
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements.  See the NOTICE file
distributed with this work for additional information
regarding copyright ownership.  The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License.  You may obtain a copy of the License at

http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

"""

import sys, os, pwd
from resource_management import *
from resource_management.libraries.functions import check_process_status
from subprocess import call

class Master(Script):
  def install(self, env):
    self.install_packages(env)
    import params
    import status_params

    Execute('echo Scriptdir is: ' + params.service_scriptsdir)

    Execute('echo installation directory is: ' + params.tweet_installdir)
    Execute('echo ambari host: ' + params.ambari_server_host)
    Execute('echo namenode host: ' + params.namenode_host)
    Execute('echo nimbus host: ' + params.nimbus_host)
    Execute('echo hive host: ' + params.hive_metastore_host)
    Execute('echo hbase host: ' + params.hbase_master_host)
    Execute('echo kafka broker: ' + params.kafka_broker_host)

    Directory(params.tweet_installdir, mode=0755, owner='root', group='root', recursive=True)
    Directory(status_params.tweet_piddir, mode=0755, owner='root', group='root', recursive=True)

    Execute('echo Copying script to ' + params.tweet_installdir)
    Execute('cp -f ' + params.service_scriptsdir + 'setup_nifi.sh ' + params.tweet_installdir)
    Execute('cp -f ' + params.service_scriptsdir + 'setup_twitter.sh ' + params.tweet_installdir)
    Execute('cp -f ' + params.service_scriptsdir + 'twitter_dashboard_v5.xml ' + params.tweet_installdir)
    self.configure(env)

  def configure(self, env):
    import params
    import status_params
    env.set_params(params)
    env.set_params(status_params)
    user_env=InlineTemplate(status_params.user_env)
    File(params.tweet_installdir + '/user-env.sh', content=user_env, owner='root',group='root')
    tweet_env=InlineTemplate(params.tweet_env)
    File(params.tweet_installdir + '/tweet-env.sh', content=tweet_env, owner='root',group='root')

  def stop(self, env):
    Execute ('echo stop')

  def start(self, env):
    import params
    import status_params
    self.configure(env)
    config_nifi_script = os.path.join(params.tweet_installdir,'setup_nifi.sh')
    nifi_template_xml = os.path.join(params.tweet_installdir,'twitter_dashboard_v5.xml')
    Execute ('pip install requests')
    if not os.path.exists(status_params.tweet_piddir):
        os.makedirs(status_params.tweet_piddir)
    #To change. Really bad!!!
    Execute ('echo Wait 30 seconds to let NiFi starts')
    Execute ('sleep 30')
    Execute ('chmod +x ' + config_nifi_script)
    Execute (config_nifi_script + ' ' + params.tweet_installdir + ' ' + nifi_template_xml + ' twitter_dashboard')
    config_twitter_script = os.path.join(params.tweet_installdir,'setup_twitter.sh')
    Execute ('chmod +x ' + config_twitter_script)
    Execute (config_twitter_script + ' ' + params.tweet_installdir + ' GetTwitter ')

  def status(self, env):
    import status_params
    self.check_flow_running(status_params.tweet_pidfile, "sandbox.hortonworks.com", "9090")

  def check_flow_running(self, pid_file, host, port):
    import requests
    if not pid_file or not os.path.isfile(pid_file):
      raise ComponentIsNotRunning()
    try:
      f = open(pid_file,"r")
      p_group_id = f.read()
      f.close()
      req = 'http://' + host + ':' + port + '/nifi-api/controller/process-groups/root/process-group-references/' + p_group_id.strip()
      r = requests.get(req)
      if r.status_code != requests.codes.ok:
        raise ComponentIsNotRunning()
      if r.json()["processGroup"]["runningCount"] < 26:
        raise ComponentIsNotRunning()
    except Exception, e:
      raise ComponentIsNotRunning()

if __name__ == "__main__":
  Master().execute()
<|endoftext|>


<commit_before>from magpie import *
from owsrequest import *
from models import find_children_by_name
from pyramid.security import Everyone as EVERYONE

class ServiceI(object):
    permission_names = []
    params_expected = []    # derived services must have 'request' at least for 'permission_requested' method
    resource_types = []

    def __init__(self, service, request):
        self.service = service
        self.request = request
        self.acl = []
        self.parser = ows_parser_factory(request)
        self.parser.parse(self.params_expected)

    @property
    def __acl__(self):
        raise NotImplementedError

    def expand_acl(self, resource, user):
        if resource:
            for ace in resource.__acl__:
                self.acl.append(ace)
            # Custom acl

            if user:
                permissions = resource.perms_for_user(user)
                for outcome, perm_user, perm_name in permission_to_pyramid_acls(permissions):
                    self.acl.append((outcome, perm_user, perm_name,))
            else:
                user = UserService.by_user_name(ANONYMOUS_USER, db_session=self.request.db)
                if user is None:
                    raise Exception('No Anonymous user in the database')
                else:
                    permissions = resource.perms_for_user(user)
                    for outcome, perm_user, perm_name in permission_to_pyramid_acls(permissions):
                        self.acl.append((outcome, EVERYONE, perm_name,))

    def permission_requested(self):
        try:
            return self.parser.params[u'request']
        except Exception as e:
            # if 'ServiceI', 'params_expected' is empty and will raise a KeyError
            raise NotImplementedError("Exception: [" + repr(e) + "]")

class ServiceWPS(ServiceI):

    permission_names = [u'getcapabilities',
                        u'describeprocess',
                        u'execute']

    params_expected = [u'service',
                       u'request',
                       u'version']

    def __init__(self, service, request):
        super(ServiceWPS, self).__init__(service, request)

    @property
    def __acl__(self):
        self.expand_acl(self.service, self.request.user)
        return self.acl

class ServiceWMS(ServiceI):

    permission_names = [u'getcapabilities',
                        u'getmap',
                        u'getfeatureinfo',
                        u'getlegendgraphic',
                        u'getmetadata']

    params_expected = [u'service',
                       u'request',
                       u'version',
                       u'layers']

    resource_types = [models.Workspace.resource_type_name]

    def __init__(self, service, request):
        super(ServiceWMS, self).__init__(service, request)

    @property
    def __acl__(self):
        self.expand_acl(self.service, self.request.user)

        #localhost:8087/geoserver/WATERSHED/wms?layers=WATERSHED:BV_1NS&request=getmap
        #localhost:8087/geoserver/wms?layers=WATERERSHED:BV1_NS&request=getmap
        #those two request lead to the same thing so, here we need to check the workspace in the layer

        #localhost:8087/geoserver/wms?request=getcapabilities (dangerous, get all workspace)
        # localhost:8087/geoserver/WATERSHED/wms?request=getcapabilities (only for the workspace in the path)
        #here we need to check the workspace in the path

        request_type = self.permission_requested()
        if request_type == 'getcapabilities':
            path_elem = self.request.path.split('/')
            wms_idx = path_elem.index('wms')
            if path_elem[wms_idx-1] != 'geoserver':
                workspace_name = path_elem[wms_idx-1]
            else:
                workspace_name = ''
        else:
            layer_name = self.parser.params['layers']
            workspace_name = layer_name.split(':')[0]

        # load workspace resource from the database
        workspace = find_children_by_name(name=workspace_name,
                                          parent_id=self.service.resource_id,
                                          db_session=self.request.db)
        if workspace:
            self.expand_acl(workspace, self.request.user)
        return self.acl

class ServiceWFS(ServiceI):

    permission_names = [u'getcapabilities',
                        u'describefeaturetype',
                        u'getfeature',
                        u'lockfeature',
                        u'transaction']

    params_expected = [u'service',
                       u'request',
                       u'version',
                       u'typenames']

    resource_types = []

    def __init__(self, service, request):
        super(ServiceWFS, self).__init__(service, request)

    @property
    def __acl__(self):
        self.expand_acl(self.service, self.request.user)
        request_type = self.permission_requested()
        if request_type == 'getcapabilities':
            path_elem = self.request.path.split('/')
            wms_idx = path_elem.index('wfs')
            if path_elem[wms_idx - 1] != 'geoserver':
                workspace_name = path_elem[wms_idx - 1]
            else:
                workspace_name = ''
        else:
            layer_name = self.parser.params['typenames']
            workspace_name = layer_name.split(':')[0]

        # load workspace resource from the database
        workspace = find_children_by_name(name=workspace_name,
                                          parent_id=self.service.resource_id,
                                          db_session=self.request.db)
        if workspace:
            self.expand_acl(workspace, self.request.user)
        return self.acl

class ServiceTHREDDS(ServiceI):

    permission_names = models.File.permission_names

    params_expected = [u'request']

    resource_types = [models.Directory.resource_type_name,
                      models.File.resource_type_name]

    def __init__(self, service, request):
        super(ServiceTHREDDS, self).__init__(service, request)

    @property
    def __acl__(self):
        self.expand_acl(self.service, self.request.user)
        elems = self.request.path.split('/')

        if 'fileServer' in elems:
            first_idx = elems.index('fileServer')
        elif 'dodsC' in elems:
            first_idx = elems.index('dodsC')
            elems[-1] = elems[-1].replace('.html', '')
        else:
            return self.acl

        elems = elems[first_idx+1::]
        db = self.request.db
        new_child = self.service
        while new_child and elems:
            name = elems.pop(0)
            new_child = find_children_by_name(name, parent_id=new_child.resource_id, db_session=db)
            self.expand_acl(new_child, self.request.user)

        return self.acl

    def permission_requested(self):
        return u'read'

    pass

service_type_dict = {u'wps': ServiceWPS,
                     u'wms': ServiceWMS,
                     u'wfs': ServiceWFS,
                     u'thredds': ServiceTHREDDS}

def service_factory(service, request):
    try:
        service_specific = service_type_dict[service.type](service, request)
        return service_specific
    except Exception as e:
        raise Exception("Failed to find requested service type. Exception: [" + repr(e) + "]")

def get_all_service_permission_names():
    all_permission_names_list = set()
    for service_type in service_type_dict.keys():
        all_permission_names_list.update(service_type_dict[service_type].permission_names)
    return all_permission_names_list
<commit_msg>support catalog with perms<commit_after>from magpie import *
from owsrequest import *
from models import find_children_by_name
from pyramid.security import Everyone as EVERYONE

class ServiceI(object):
    permission_names = []
    params_expected = []    # derived services must have 'request' at least for 'permission_requested' method
    resource_types = []

    def __init__(self, service, request):
        self.service = service
        self.request = request
        self.acl = []
        self.parser = ows_parser_factory(request)
        self.parser.parse(self.params_expected)

    @property
    def __acl__(self):
        raise NotImplementedError

    def expand_acl(self, resource, user):
        if resource:
            for ace in resource.__acl__:
                self.acl.append(ace)
            # Custom acl

            if user:
                permissions = resource.perms_for_user(user)
                for outcome, perm_user, perm_name in permission_to_pyramid_acls(permissions):
                    self.acl.append((outcome, perm_user, perm_name,))
            else:
                user = UserService.by_user_name(ANONYMOUS_USER, db_session=self.request.db)
                if user is None:
                    raise Exception('No Anonymous user in the database')
                else:
                    permissions = resource.perms_for_user(user)
                    for outcome, perm_user, perm_name in permission_to_pyramid_acls(permissions):
                        self.acl.append((outcome, EVERYONE, perm_name,))

    def permission_requested(self):
        try:
            return self.parser.params[u'request']
        except Exception as e:
            # if 'ServiceI', 'params_expected' is empty and will raise a KeyError
            raise NotImplementedError("Exception: [" + repr(e) + "]")

class ServiceWPS(ServiceI):

    permission_names = [u'getcapabilities',
                        u'describeprocess',
                        u'execute']

    params_expected = [u'service',
                       u'request',
                       u'version']

    def __init__(self, service, request):
        super(ServiceWPS, self).__init__(service, request)

    @property
    def __acl__(self):
        self.expand_acl(self.service, self.request.user)
        return self.acl

class ServiceWMS(ServiceI):

    permission_names = [u'getcapabilities',
                        u'getmap',
                        u'getfeatureinfo',
                        u'getlegendgraphic',
                        u'getmetadata']

    params_expected = [u'service',
                       u'request',
                       u'version',
                       u'layers']

    resource_types = [models.Workspace.resource_type_name]

    def __init__(self, service, request):
        super(ServiceWMS, self).__init__(service, request)

    @property
    def __acl__(self):
        self.expand_acl(self.service, self.request.user)

        #localhost:8087/geoserver/WATERSHED/wms?layers=WATERSHED:BV_1NS&request=getmap
        #localhost:8087/geoserver/wms?layers=WATERERSHED:BV1_NS&request=getmap
        #those two request lead to the same thing so, here we need to check the workspace in the layer

        #localhost:8087/geoserver/wms?request=getcapabilities (dangerous, get all workspace)
        # localhost:8087/geoserver/WATERSHED/wms?request=getcapabilities (only for the workspace in the path)
        #here we need to check the workspace in the path

        request_type = self.permission_requested()
        if request_type == 'getcapabilities':
            path_elem = self.request.path.split('/')
            wms_idx = path_elem.index('wms')
            if path_elem[wms_idx-1] != 'geoserver':
                workspace_name = path_elem[wms_idx-1]
            else:
                workspace_name = ''
        else:
            layer_name = self.parser.params['layers']
            workspace_name = layer_name.split(':')[0]

        # load workspace resource from the database
        workspace = find_children_by_name(name=workspace_name,
                                          parent_id=self.service.resource_id,
                                          db_session=self.request.db)
        if workspace:
            self.expand_acl(workspace, self.request.user)
        return self.acl

class ServiceWFS(ServiceI):

    permission_names = [u'getcapabilities',
                        u'describefeaturetype',
                        u'getfeature',
                        u'lockfeature',
                        u'transaction']

    params_expected = [u'service',
                       u'request',
                       u'version',
                       u'typenames']

    resource_types = []

    def __init__(self, service, request):
        super(ServiceWFS, self).__init__(service, request)

    @property
    def __acl__(self):
        self.expand_acl(self.service, self.request.user)
        request_type = self.permission_requested()
        if request_type == 'getcapabilities':
            path_elem = self.request.path.split('/')
            wms_idx = path_elem.index('wfs')
            if path_elem[wms_idx - 1] != 'geoserver':
                workspace_name = path_elem[wms_idx - 1]
            else:
                workspace_name = ''
        else:
            layer_name = self.parser.params['typenames']
            workspace_name = layer_name.split(':')[0]

        # load workspace resource from the database
        workspace = find_children_by_name(name=workspace_name,
                                          parent_id=self.service.resource_id,
                                          db_session=self.request.db)
        if workspace:
            self.expand_acl(workspace, self.request.user)
        return self.acl

class ServiceTHREDDS(ServiceI):

    permission_names = models.File.permission_names

    params_expected = [u'request']

    resource_types = [models.Directory.resource_type_name,
                      models.File.resource_type_name]

    def __init__(self, service, request):
        super(ServiceTHREDDS, self).__init__(service, request)

    @property
    def __acl__(self):
        self.expand_acl(self.service, self.request.user)
        elems = self.request.path.split('/')

        if 'fileServer' in elems:
            first_idx = elems.index('fileServer')
        elif 'dodsC' in elems:
            first_idx = elems.index('dodsC')
            elems[-1] = elems[-1].replace('.html', '')
        elif 'catalog' in elems:
            first_idx = elems.index('catalog')
        else:
            return self.acl

        elems = elems[first_idx+1::]
        db = self.request.db
        new_child = self.service
        while new_child and elems:
            name = elems.pop(0)
            new_child = find_children_by_name(name, parent_id=new_child.resource_id, db_session=db)
            self.expand_acl(new_child, self.request.user)

        return self.acl

    def permission_requested(self):
        return u'read'

    pass

service_type_dict = {u'wps': ServiceWPS,
                     u'wms': ServiceWMS,
                     u'wfs': ServiceWFS,
                     u'thredds': ServiceTHREDDS}

def service_factory(service, request):
    try:
        service_specific = service_type_dict[service.type](service, request)
        return service_specific
    except Exception as e:
        raise Exception("Failed to find requested service type. Exception: [" + repr(e) + "]")

def get_all_service_permission_names():
    all_permission_names_list = set()
    for service_type in service_type_dict.keys():
        all_permission_names_list.update(service_type_dict[service_type].permission_names)
    return all_permission_names_list
<|endoftext|>


<commit_before>#!/usr/bin/python
# -*- coding: iso-8859-15 -*-
#
# script to read xrf sensor data from serial
# validate message to avoid carbage being pushed to the db
# writes also last detected temp value from each sensor to
# a separate file
#
# This code is put together from various examples in the net
# If you see this usefull in your project, use it. :)
#
# jukkalau@gmail.com
#

import serial
import re
import sqlite3
import logging
import logging.handlers
import ConfigParser

my_logger = logging.getLogger('MyLogger')
my_logger.setLevel(logging.DEBUG)
handler = logging.handlers.SysLogHandler(address = '/dev/log')
my_logger.addHandler(handler)

parser = ConfigParser.SafeConfigParser()
parser.read('config.ini')

err = 0 #error counter, hackish way to recover from offsync in xrf communication
baud = 9600
port = '/dev/ttyAMA0'
ser = serial.Serial(port, baud)
con = sqlite3.connect('temperature.db')

def main():
    while True:
        with con:
            cur = con.cursor()
            llapMsg = str(ser.read(12))
            my_logger.debug(llapMsg)
            #match voltage message
            #matches also negative value aBBTMPA-9.99
            validator = re.compile("a([A-Z][A-Z])TMPA(( |-)?\d.{1,3}\d)")
            m = validator.match(llapMsg)
            if m:
                sqlstring = "insert into temps(sensor,value) values('" + m.group(1)\
                + "','" + m.group(2) + "')"
                my_logger.debug(sqlstring)
                cur.execute(sqlstring)
                fh = open("www/temp/last_temp" + m.group(1) + ".txt", "w+")
                sname = parser.get(m.group(1),"displayName")
                # fetch calibration multiplier from ini file and use that to
                # store fixed temp reading to db
                fixedValue = float(parser.get(m.group(1),"calibrationValue")) \
                * float(m.group(2))
                fh.write(sname + ":" + str(round(fixedValue, 2)))
                fh.close()
                err = 0
            # todo: fix voltage reporting to be done in correct sensor file
            elif re.search("BATT", llapMsg):
                value = llapMsg[7:12]
                fh = open("www/temp/last_temp" + llapMsg[1:3] +".txt" , "w+")
                fh.write("Patteri " + llapMsg[1:3] + ": " + value + "V")
                fh.close()
                err = 0
            elif re.search("AWAKE", llapMsg):
                my_logger.debug("AWAKE message catched")
	        err = 0
            else:
                if err > 10:
                    my_logger.debug("Error count hit, bailing out")
                    return 1
                my_logger.debug(
                    "reg-exp did not catch message, emptying serial buffer")
                null = ser.read(24)
                err = e + 1

main()
<commit_msg>added initial support for humidity sensor<commit_after>#!/usr/bin/python
# -*- coding: iso-8859-15 -*-
#
# script to read xrf sensor data from serial
# validate message to avoid carbage being pushed to the db
# writes also last detected temp value from each sensor to
# a separate file
#
# This code is put together from various examples in the net
# If you see this usefull in your project, use it. :)
#
# jukkalau@gmail.com
#

import serial
import re
import sqlite3
import logging
import logging.handlers
import ConfigParser

my_logger = logging.getLogger('MyLogger')
my_logger.setLevel(logging.DEBUG)
handler = logging.handlers.SysLogHandler(address = '/dev/log')
my_logger.addHandler(handler)

parser = ConfigParser.SafeConfigParser()
parser.read('config.ini')

baud = 9600
port = '/dev/ttyAMA0'
ser = serial.Serial(port, baud)
con = sqlite3.connect('temperature.db')

def main():
    err = 0 # error counter, hackish way to recover from offsync in xrf communication
    while True:
        with con:
            cur = con.cursor()
            llapMsg = str(ser.read(12))
            my_logger.debug(llapMsg)
            #match voltage message
            #matches also negative value aBBTMPA-9.99
            validatorTemp = re.compile("a([A-Z][A-Z])TMPA(( |-)?\d.{1,3}\d)")
            validatorHum = re.compile("a([A-Z][A-Z])HUMA(( |-)?\d.{1,3}\d)")
            m = validatorTemp.match(llapMsg)
            mHum = validatorHum.match(llapMsg)
            if m:
                sqlstring = "insert into temps(sensor,value) values('" + m.group(1)\
                + "','" + m.group(2) + "')"
                my_logger.debug(sqlstring)
                cur.execute(sqlstring)
                fh = open("www/temp/last_temp" + m.group(1) + ".txt", "w+")
                sname = parser.get(m.group(1),"displayName")
                # fetch calibration multiplier from ini file and use that to
                # store fixed temp reading to db
                fixedValue = float(parser.get(m.group(1),"calibrationValue")) \
                * float(m.group(2))
                fh.write(sname + ":" + str(round(fixedValue, 2)))
                fh.close()
                err = 0
            elif mHum:  # handle humidity sendor ( by cheating )
                # todo: add new table to db to hold humidity values
                # now, only 2nd identifier character being H differentiates temp from
                # humidity
                sqlstring = "insert into temps(sensor,value) values('" + mHum.group(1)\
                + "','" + mHum.group(2) + "')"
                my_logger.debug(sqlstring)
                cur.execute(sqlstring)
                fh = open("www/temp/last_hum" + mHum.group(1) + ".txt", "w+")
                sname = parser.get(mHum.group(1),"displayName")
                # fetch calibration multiplier from ini file and use that to
                # store fixed temp reading to db
                fixedValue = float(parser.get(mHum.group(1),"calibrationValue")) \
                * float(mHum.group(2))
                fh.write(sname + ":" + str(round(fixedValue, 2)))
                fh.close()
                err = 0
            # todo: fix voltage reporting to be done in correct sensor file
            elif re.search("BATT", llapMsg):
                value = llapMsg[7:12]
                fh = open("www/temp/last_temp" + llapMsg[1:3] +".txt" , "w+")
                fh.write("Patteri " + llapMsg[1:3] + ": " + value + "V")
                fh.close()
                err = 0
            elif re.search("AWAKE", llapMsg):
                my_logger.debug("AWAKE message catched")
	        err = 0
            elif re.search("STARTED", llapMsg):
                my_logger.debug("STARTED message catched")
                err = 0
            elif re.search("SLEEPING", llapMsg):
                my_logger.debug("STARTED message catched")
                err = 0
            else:
                if err > 13:
                    my_logger.debug("Error count hit, bailing out")
                    return 1
                my_logger.debug(
                    "reg-exp did not catch message, reading extra char from serial buffer")
                crap = ser.read(1)
                my_logger.debug(crap)
                err = err + 1

main()
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

import rospy
import mongodb_store_msgs.srv as dc_srv
import mongodb_store.util as dc_util
from mongodb_store.message_store import MessageStoreProxy
from strands_executive_msgs.msg import Task, TaskEvent
from datetime import datetime, timedelta
from task_executor import task_routine, task_query
from task_executor.utils import rostime_to_python

import argparse

if __name__ == '__main__':

    rospy.init_node("routine_analysis")

    msg_store = MessageStoreProxy(collection='task_events')

    parser = argparse.ArgumentParser(description='Analyses the task execution behaviour in a routine window. Assumes all task ids are unique in this window.')
    parser.add_argument('start', metavar='S', type=task_query.mkdatetime, nargs='?',
                   help='start datetime of query, defaults to start of last routine. Formatted "d/m/y H:M" e.g. "06/07/14 06:38"')

    parser.add_argument('end', metavar='E', type=task_query.mkdatetime, nargs='?',
                   help='end datetime of query, defaults to end of last routine or now for an unclosed routine. Formatted "d/m/y H:M" e.g. "06/07/14 06:38"')

    parser.add_argument('-t', '--tasks', type=int, default=1, nargs='?',
                    help='Number of tasks required for a routine to be considered')

    args = parser.parse_args()

    try:

        results = task_query.query_tasks(msg_store, event=[TaskEvent.ROUTINE_STARTED, TaskEvent.ROUTINE_STOPPED])

        # make sure we start with a ROUTINE_STARTED
        while results[0][0].event == TaskEvent.ROUTINE_STOPPED:
            del results[0]

        routines = []
        start = None
        end = None

        # make sure we just have the pairs which are start/end
        for e in results:
            event = e[0]
            if event.event == TaskEvent.ROUTINE_STARTED:
                start = event
            elif event.event == TaskEvent.ROUTINE_STOPPED:
                assert start is not None
                routines.append((start, event))
                start = None

        filtered_routines = []
        for i in range(len(routines)):
            start = rostime_to_python(routines[i][0].time)
            end = rostime_to_python(routines[i][1].time)
            results = task_query.query_tasks(msg_store,
                        start_date=start,
                        end_date=end,
                        event=[TaskEvent.TASK_STARTED]
                        )

            if len(results) >= args.tasks:
                filtered_routines.append(routines[i])
                print 'routine %s: %s to %s, duration: %s, tasks: %s' % (len(filtered_routines)-1, start, end, end-start, len(results))

        # if args.start is None:
        #     # find latest routine start
        #     results = task_query.query_tasks(msg_store, event=TaskEvent.ROUTINE_STARTED)
        #     assert len(results) > 0
        #     start = rostime_to_python(results[-1][0].time)
        # else:
        #     start = args.start

        # end = args.end
        # if end is None:
        #     results = task_query.query_tasks(msg_store, event=TaskEvent.ROUTINE_STOPPED)
        #     if len(results) > 0:
        #         end_routine = rostime_to_python(results[-1][0].time)
        #         if start is not None and end_routine > start:
        #             end = end_routine

        # results = task_query.query_tasks(msg_store,
        #                 start_date=start,
        #                 end_date=end,
        #                 event=[TaskEvent.TASK_STARTED, TaskEvent.TASK_FINISHED, TaskEvent.NAVIGATION_SUCCEEDED]
        #                 )

        # print 'Tasks Start: %s' % start
        # print 'Tasks End: %s' % end
        # task_query.summarise(results)

    except rospy.ServiceException, e:
        print "Service call failed: %s"%e

<commit_msg>Turned script into interactive cmds<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

import rospy
import mongodb_store_msgs.srv as dc_srv
import mongodb_store.util as dc_util
from mongodb_store.message_store import MessageStoreProxy
from strands_executive_msgs.msg import Task, TaskEvent
from datetime import datetime, timedelta
from task_executor import task_routine, task_query
from task_executor.utils import rostime_to_python

import argparse
import cmd

class RoutineAnalyser(cmd.Cmd):

    def __init__(self, routine_pairs):
        cmd.Cmd.__init__(self)
        # super(RoutineAnalyser, self).__init__()
        self.routine_pairs = routine_pairs

    def do_merge(self, idx):
        try:
            idx = int(idx)
            if idx >= 0 and idx < len(self.routine_pairs) - 1:
                print 'merging %s into %s' % (idx, idx+1)
                self.routine_pairs[idx+1] = (self.routine_pairs[idx][0], self.routine_pairs[idx+1][1])
                del self.routine_pairs[idx]
            else:
                print 'invalid routine index, valid range from 0 to %s' % (len(self.routine_pairs) - 2)
        except ValueError, e:
            print 'provided argument was not an int: %s' % idx

    def do_print(self, line):
        for i in range(len(self.routine_pairs)):
            start = rostime_to_python(self.routine_pairs[i][0].time)
            end = rostime_to_python(self.routine_pairs[i][1].time)
            results = task_query.query_tasks(msg_store,
                        start_date=start,
                        end_date=end,
                        event=[TaskEvent.TASK_STARTED]
                        )

            print 'routine %s: %s to %s, duration: %s, tasks: %s' % (i, start, end, end-start, len(results))

    def help_greet(self):
        print '\n'.join([ 'print', 'Print the available routines'])

    def help_merge(self):
        print '\n'.join([ 'merge [idx]', 'Merge the routine at idx into the routine after it'])

    def do_EOF(self, line):
        return True

if __name__ == '__main__':

    rospy.init_node("routine_analysis")

    msg_store = MessageStoreProxy(collection='task_events')

    parser = argparse.ArgumentParser(description='Analyses the task execution behaviour in a routine window. Assumes all task ids are unique in this window.')
    parser.add_argument('start', metavar='S', type=task_query.mkdatetime, nargs='?',
                   help='start datetime of query, defaults to start of last routine. Formatted "d/m/y H:M" e.g. "06/07/14 06:38"')

    parser.add_argument('end', metavar='E', type=task_query.mkdatetime, nargs='?',
                   help='end datetime of query, defaults to end of last routine or now for an unclosed routine. Formatted "d/m/y H:M" e.g. "06/07/14 06:38"')

    parser.add_argument('-t', '--tasks', type=int, default=1, nargs='?',
                    help='Number of tasks required for a routine to be considered')

    args = parser.parse_args()

    try:

        results = task_query.query_tasks(msg_store, event=[TaskEvent.ROUTINE_STARTED, TaskEvent.ROUTINE_STOPPED])

        # make sure we start with a ROUTINE_STARTED
        while results[0][0].event == TaskEvent.ROUTINE_STOPPED:
            del results[0]

        routines = []
        start = None
        end = None

        # make sure we just have the pairs which are start/end
        for e in results:
            event = e[0]
            if event.event == TaskEvent.ROUTINE_STARTED:
                start = event
            elif event.event == TaskEvent.ROUTINE_STOPPED:
                assert start is not None
                routines.append((start, event))
                start = None

        filtered_routines = []
        for i in range(len(routines)):
            start = rostime_to_python(routines[i][0].time)
            end = rostime_to_python(routines[i][1].time)
            results = task_query.query_tasks(msg_store,
                        start_date=start,
                        end_date=end,
                        event=[TaskEvent.TASK_STARTED]
                        )

            if len(results) >= args.tasks:
                filtered_routines.append(routines[i])
                # print 'routine %s: %s to %s, duration: %s, tasks: %s' % (len(filtered_routines)-1, start, end, end-start, len(results))

        RoutineAnalyser(filtered_routines).cmdloop()

        # if args.start is None:
        #     # find latest routine start
        #     results = task_query.query_tasks(msg_store, event=TaskEvent.ROUTINE_STARTED)
        #     assert len(results) > 0
        #     start = rostime_to_python(results[-1][0].time)
        # else:
        #     start = args.start

        # end = args.end
        # if end is None:
        #     results = task_query.query_tasks(msg_store, event=TaskEvent.ROUTINE_STOPPED)
        #     if len(results) > 0:
        #         end_routine = rostime_to_python(results[-1][0].time)
        #         if start is not None and end_routine > start:
        #             end = end_routine

        # results = task_query.query_tasks(msg_store,
        #                 start_date=start,
        #                 end_date=end,
        #                 event=[TaskEvent.TASK_STARTED, TaskEvent.TASK_FINISHED, TaskEvent.NAVIGATION_SUCCEEDED]
        #                 )

        # print 'Tasks Start: %s' % start
        # print 'Tasks End: %s' % end
        # task_query.summarise(results)

    except rospy.ServiceException, e:
        print "Service call failed: %s"%e

<|endoftext|>


<commit_before>#!/usr/bin/python
import os
import sched, time
import math
import pygame, sys
from pygame.locals import *
import RPi.GPIO as GPIO
from flowmeter import *
from TemperatureController import *
import threading

# ===========================================================================
# Define Pin setup
# ===========================================================================
FLOWSENSOR_PIN = 4
TEMPERATURESENSOR_PIN = 22
FRIDGEPOWER_PIN = 23
boardRevision = GPIO.RPI_REVISION
GPIO.cleanup()
GPIO.setmode(GPIO.BCM)
GPIO.setup(FLOWSENSOR_PIN,GPIO.IN, pull_up_down=GPIO.PUD_UP)
GPIO.setup(FRIDGEPOWER_PIN, GPIO.OUT)
GPIO.output(FRIDGEPOWER_PIN, GPIO.LOW)

# ===========================================================================
# Define Pin setup
# ===========================================================================
pygame.init()
VIEW_WIDTH = 1248
VIEW_HEIGHT = 688
BLACK = (0,0,0)
WHITE = (255,255,255)
pygame.display.set_caption('')
windowSurface = pygame.display.set_mode([600,400])
windowInfo = pygame.display.Info()
FONTSIZE = 48
LINEHEIGHT = 32
basicFont = pygame.font.SysFont(None, FONTSIZE)

#set up flow meteru k
fm = FlowMeter('metric')
#set up temperature controller
tc = TemperatureController(22)
bRun = False
#This gets run whenever an interrupt triggers it due to pin 4 being grounded.
def doAClick(channel):
  currentTime = int(time.time() * FlowMeter.MS_IN_A_SECOND)
  fm.update(currentTime)
  print(fm.totalPour)

GPIO.add_event_detect(4, GPIO.RISING, callback=doAClick, bouncetime=20)

def render():
  windowSurface.fill(BLACK)

  text = basicFont.render('Temperature: ' + tc.GetFormattedTemperature(), True, WHITE, BLACK)
  textRect= text.get_rect()
  windowSurface.blit(text, (40,6*LINEHEIGHT))

  text = basicFont.render('Amount Poured(L)' + fm.getFormattedTotalPour(), True, WHITE, BLACK)
  textRect = text.get_rect()
  windowSurface.blit(text, (40,7*LINEHEIGHT))

  text = basicFont.render('Current Pour' + fm.getFormattedThisPour() , True, WHITE, BLACK)
  textRect = text.get_rect()
  windowSurface.blit(text, (40,8*LINEHEIGHT))

  #Display everything
  pygame.display.flip()

# Operate the fridge based off the temp read
def FridgeControl(tc):
	if (tc.temperature >= 1):
		#Turn fridge On
		GPIO.output(FRIDGE_PIN, GPIO.HIGH)
		print ("ON")
		bFridgeOn = True
	elif (tc.temperature <= -1 ):
		#Turn fridge off
		GPIO.output(FRIDGE_PIN, GPIO.LOW)
		bFridgeOn = False

def ReadTemp(tc):
    while bRun:
        tc.read_dht22()
    time.sleep(5)

tTemp = threading.Thread(target=ReadTemp, args=(tc,15))
tTemp.start()
while True:
	if  ( tc.temperature > -254):
		FridgeControl(tc)
	for event in pygame.event.get():
        if event.type == QUIT or (event.type == KEYUP and event.key == K_ESCAPE):
            pygame.quit()
            tTemp.stop()
            #Cleanup GPIO
            GPIO.cleanup()
            sys.exit()

  currentTime = int(time.time() * FlowMeter.MS_IN_A_SECOND)
  if(fm.thisPour > 0.23 and currentTime - fm.lastClick > 10000):
    fm.thisPour = 0.0
	render()
<commit_msg>mainsimple<commit_after>#!/usr/bin/python
import os
import sched, time
import math
import pygame, sys
from pygame.locals import *
import RPi.GPIO as GPIO
from flowmeter import *
from TemperatureController import *
import threading

# ===========================================================================
# Define Pin setup
# ===========================================================================
FLOWSENSOR_PIN = 4
TEMPERATURESENSOR_PIN = 22
FRIDGEPOWER_PIN = 23
boardRevision = GPIO.RPI_REVISION
GPIO.cleanup()
GPIO.setmode(GPIO.BCM)
GPIO.setup(FLOWSENSOR_PIN,GPIO.IN, pull_up_down=GPIO.PUD_UP)
GPIO.setup(FRIDGEPOWER_PIN, GPIO.OUT)
GPIO.output(FRIDGEPOWER_PIN, GPIO.LOW)

# ===========================================================================
# Define Pin setup
# ===========================================================================
pygame.init()
VIEW_WIDTH = 1248
VIEW_HEIGHT = 688
BLACK = (0,0,0)
WHITE = (255,255,255)
pygame.display.set_caption('')
windowSurface = pygame.display.set_mode([600,400])
windowInfo = pygame.display.Info()
FONTSIZE = 48
LINEHEIGHT = 32
basicFont = pygame.font.SysFont(None, FONTSIZE)

#set up flow meteru k
fm = FlowMeter('metric')
#set up temperature controller
tc = TemperatureController(22)
bRun = False
#This gets run whenever an interrupt triggers it due to pin 4 being grounded.
def doAClick(channel):
    currentTime = int(time.time() * FlowMeter.MS_IN_A_SECOND)
    fm.update(currentTime)
    print(fm.totalPour)

GPIO.add_event_detect(4, GPIO.RISING, callback=doAClick, bouncetime=20)

def render():
    windowSurface.fill(BLACK)

    text = basicFont.render('Temperature: ' + tc.GetFormattedTemperature(), True, WHITE, BLACK)
    textRect= text.get_rect()
    windowSurface.blit(text, (40,6*LINEHEIGHT))

    text = basicFont.render('Amount Poured(L)' + fm.getFormattedTotalPour(), True, WHITE, BLACK)
    textRect = text.get_rect()
    windowSurface.blit(text, (40,7*LINEHEIGHT))

    text = basicFont.render('Current Pour' + fm.getFormattedThisPour() , True, WHITE, BLACK)
    textRect = text.get_rect()
    windowSurface.blit(text, (40,8*LINEHEIGHT))

    #Display everything
    pygame.display.flip()

# Operate the fridge based off the temp read
def FridgeControl(tc):
	if (tc.temperature >= 1):
		#Turn fridge On
		GPIO.output(FRIDGE_PIN, GPIO.HIGH)
		print ("ON")
		bFridgeOn = True
	elif (tc.temperature <= -1 ):
		#Turn fridge off
		GPIO.output(FRIDGE_PIN, GPIO.LOW)
		bFridgeOn = False

def ReadTemp(tc):
    while bRun:
        tc.read_dht22()
    time.sleep(5)

tTemp = threading.Thread(target=ReadTemp, args=(tc,15))
tTemp.start()
while True:
	if  ( tc.temperature > -254):
        FridgeControl(tc)
	for event in pygame.event.get():
        if event.type == QUIT or (event.type == KEYUP and event.key == K_ESCAPE):
            pygame.quit()
            tTemp.stop()
            #Cleanup GPIO
            GPIO.cleanup()
            sys.exit()

    currentTime = int(time.time() * FlowMeter.MS_IN_A_SECOND)
    if(fm.thisPour > 0.23 and currentTime - fm.lastClick > 10000):
        fm.thisPour = 0.0
	render()
<|endoftext|>


<commit_before>from time import time, sleep
from unittest import TestCase

import stubs

from mediator import Mediator, Event, SubscriberInterface

class Listener(object):
    def __init__(self):
        self.events = []

    def __call__(self, event):
        self.events.append(event)

class EventOne(Event):
    event_name = 'event_one'

class EventTwo(Event):
    event_name = 'event_two'

class EventThree(Event):
    event_name = 'event_three'

class Subscriber(SubscriberInterface):
    def __init__(self):
        self.results = {}

    def first(self, event):
        self.results['first'] = (event, time())

    def middle(self, event):
        sleep(0.01)
        self.results['middle'] = (event, time())

    def last(self, event):
        sleep(0.01)
        self.results['last'] = (event, time())

    def another_event_handler(self, event):
        self.results['another'] = event

    def even_more_event_handler(self, event):
        self.results['even_more'] = event

    def get_subscribed_events(self):
        return {
            'event_one': [
                ['middle'],
                ['first', -255],
                ['last', 255],
            ],
            'event_two': 'another_event_handler',
            'event_three': ['even_more_event_handler']
        }

class BadSubscriberOne(SubscriberInterface):
    def get_subscribed_events(self):
        return {'test_event': None}

class BadSubscriberTwo(SubscriberInterface):
    def get_subscribed_events(self):
        return {'test_event': []}

class TestMediator(TestCase):
    def test_dispatch_failed(self):
        meditator = Mediator()
        with self.assertRaises(TypeError) as context:
            meditator.dispatch('unexpected_event')
        self.assertEqual(str(context.exception), 'Expects instance of Event')

    def test_add_listener_with_object(self):
        mediator = Mediator()
        listener = Listener()
        mediator.add_listener(Event, listener)
        event = Event()
        mediator.dispatch(event)
        self.assertEqual(len(listener.events), 1)
        self.assertIs(listener.events[0], event)

    def test_add_listener_with_string(self):
        mediator = Mediator()
        listener = Listener()
        mediator.add_listener('Event', listener)
        event = Event()
        event = mediator.dispatch(event)
        self.assertEqual(len(listener.events), 1)
        self.assertIs(listener.events[0], event)

    def test_add_listener_with_same_priority_failed(self):
        mediator = Mediator()
        mediator.add_listener(Event, lambda event: None, 0)
        with self.assertRaises(IndexError):
            mediator.add_listener(Event, lambda event: None, 0)

    def test_add_listener_with_invalid_event_failed(self):
        mediator = Mediator()
        listener = Listener()
        with self.assertRaises(TypeError) as context:
            mediator.add_listener(1, listener)
        self.assertEqual(str(context.exception), 'Expects subclass of Event or str')

    def test_remove_listener(self):
        mediator = Mediator()
        listener1 = Listener()
        listener2 = Listener()
        mediator.add_listener(Event, listener1)
        mediator.add_listener(Event, listener2)
        event = Event()
        mediator.dispatch(event)
        mediator.remove_listener(Event, listener1)
        mediator.dispatch(event)
        self.assertEqual(len(listener1.events), 1)
        self.assertEqual(len(listener2.events), 2)
        mediator.remove_listener(Event)
        mediator.dispatch(event)
        self.assertEqual(len(listener1.events), 1)
        self.assertEqual(len(listener2.events), 2)

    def test_remove_listener_with_invalid_event_failed(self):
        mediator = Mediator()
        listener = Listener()
        with self.assertRaises(TypeError) as context:
            mediator.remove_listener(1, listener)
        self.assertEqual(str(context.exception), 'Expects subclass of Event or str')

    def test_add_subscriber(self):
        mediator = Mediator()

        with self.assertRaises(TypeError) as context:
            mediator.add_subscriber(object())
        self.assertEqual(str(context.exception), 'Expects instance of SubscriberInterface')

        subscriber = Subscriber()
        mediator.add_subscriber(subscriber)

        event = EventOne()
        mediator.dispatch(event)
        self.assertEqual(len(subscriber.results), 3)
        events = [result[0] for result in subscriber.results.values()]
        self.assertIs(events[0], events[1])
        self.assertIs(events[1], events[2])
        first, middle, last = (
            subscriber.results['first'][1],
            subscriber.results['middle'][1],
            subscriber.results['last'][1]
        )
        self.assertTrue(first < middle < last, '%r %r %r' % (first, middle, last))

        event = EventTwo()
        mediator.dispatch(event)
        self.assertIs(subscriber.results['another'], event)

        event = EventThree()
        mediator.dispatch(event)
        self.assertIs(subscriber.results['even_more'], event)

        with self.assertRaises(ValueError):
            mediator.add_subscriber(BadSubscriberOne())

        with self.assertRaises(ValueError):
            mediator.add_subscriber(BadSubscriberTwo())

class TestSubscriberInterface(TestCase):
    def test_get_subscribed_events(self):
        subscriber = SubscriberInterface()
        with self.assertRaises(NotImplementedError):
            subscriber.get_subscribed_events()

class TestEvent(TestCase):
    def test_get_event_name(self):
        self.assertEqual(Event.get_event_name(), 'Event')
        self.assertEqual(EventOne.get_event_name(), 'event_one')
        self.assertEqual(EventTwo.get_event_name(), 'event_two')
        self.assertEqual(EventThree.get_event_name(), 'event_three')

class TestEventDecorator(TestCase):
    def test_event_decorator(self):
        mediator = Mediator()
        event = stubs.VenusianEvent()
        mediator.dispatch(event)
        self.assertFalse(event.success)
        mediator.scan(package=stubs)
        mediator.dispatch(event)
        self.assertTrue(event.success)
<commit_msg>Add a test for multi inheritance fix<commit_after>from time import time, sleep
from unittest import TestCase

import stubs

from mediator import Mediator, Event, SubscriberInterface

class Listener(object):
    def __init__(self):
        self.events = []

    def __call__(self, event):
        self.events.append(event)

class EventOne(Event):
    event_name = 'event_one'

class EventTwo(Event):
    event_name = 'event_two'

class EventThree(Event):
    event_name = 'event_three'

class Subscriber(SubscriberInterface):
    def __init__(self):
        self.results = {}

    def first(self, event):
        self.results['first'] = (event, time())

    def middle(self, event):
        sleep(0.01)
        self.results['middle'] = (event, time())

    def last(self, event):
        sleep(0.01)
        self.results['last'] = (event, time())

    def another_event_handler(self, event):
        self.results['another'] = event

    def even_more_event_handler(self, event):
        self.results['even_more'] = event

    def get_subscribed_events(self):
        return {
            'event_one': [
                ['middle'],
                ['first', -255],
                ['last', 255],
            ],
            'event_two': 'another_event_handler',
            'event_three': ['even_more_event_handler']
        }

class BadSubscriberOne(SubscriberInterface):
    def get_subscribed_events(self):
        return {'test_event': None}

class BadSubscriberTwo(SubscriberInterface):
    def get_subscribed_events(self):
        return {'test_event': []}

class TestMediator(TestCase):
    def test_dispatch_failed(self):
        meditator = Mediator()
        with self.assertRaises(TypeError) as context:
            meditator.dispatch('unexpected_event')
        self.assertEqual(str(context.exception), 'Expects instance of Event')

    def test_add_listener_with_object(self):
        mediator = Mediator()
        listener = Listener()
        mediator.add_listener(Event, listener)
        event = Event()
        mediator.dispatch(event)
        self.assertEqual(len(listener.events), 1)
        self.assertIs(listener.events[0], event)

    def test_add_listener_with_string(self):
        mediator = Mediator()
        listener = Listener()
        mediator.add_listener('Event', listener)
        event = Event()
        event = mediator.dispatch(event)
        self.assertEqual(len(listener.events), 1)
        self.assertIs(listener.events[0], event)

    def test_add_listener_with_same_priority_failed(self):
        mediator = Mediator()
        mediator.add_listener(Event, lambda event: None, 0)
        with self.assertRaises(IndexError):
            mediator.add_listener(Event, lambda event: None, 0)

    def test_add_listener_with_invalid_event_failed(self):
        mediator = Mediator()
        listener = Listener()
        with self.assertRaises(TypeError) as context:
            mediator.add_listener(1, listener)
        self.assertEqual(str(context.exception), 'Expects subclass of Event or str')

    def test_remove_listener(self):
        mediator = Mediator()
        listener1 = Listener()
        listener2 = Listener()
        mediator.add_listener(Event, listener1)
        mediator.add_listener(Event, listener2)
        event = Event()
        mediator.dispatch(event)
        mediator.remove_listener(Event, listener1)
        mediator.dispatch(event)
        self.assertEqual(len(listener1.events), 1)
        self.assertEqual(len(listener2.events), 2)
        mediator.remove_listener(Event)
        mediator.dispatch(event)
        self.assertEqual(len(listener1.events), 1)
        self.assertEqual(len(listener2.events), 2)

    def test_remove_listener_with_invalid_event_failed(self):
        mediator = Mediator()
        listener = Listener()
        with self.assertRaises(TypeError) as context:
            mediator.remove_listener(1, listener)
        self.assertEqual(str(context.exception), 'Expects subclass of Event or str')

    def test_add_subscriber(self):
        mediator = Mediator()

        with self.assertRaises(TypeError) as context:
            mediator.add_subscriber(object())
        self.assertEqual(str(context.exception), 'Expects instance of SubscriberInterface')

        subscriber = Subscriber()
        mediator.add_subscriber(subscriber)

        event = EventOne()
        mediator.dispatch(event)
        self.assertEqual(len(subscriber.results), 3)
        events = [result[0] for result in subscriber.results.values()]
        self.assertIs(events[0], events[1])
        self.assertIs(events[1], events[2])
        first, middle, last = (
            subscriber.results['first'][1],
            subscriber.results['middle'][1],
            subscriber.results['last'][1]
        )
        self.assertTrue(first < middle < last, '%r %r %r' % (first, middle, last))

        event = EventTwo()
        mediator.dispatch(event)
        self.assertIs(subscriber.results['another'], event)

        event = EventThree()
        mediator.dispatch(event)
        self.assertIs(subscriber.results['even_more'], event)

        with self.assertRaises(ValueError):
            mediator.add_subscriber(BadSubscriberOne())

        with self.assertRaises(ValueError):
            mediator.add_subscriber(BadSubscriberTwo())

class TestSubscriberInterface(TestCase):
    def test_get_subscribed_events(self):
        subscriber = SubscriberInterface()
        with self.assertRaises(NotImplementedError):
            subscriber.get_subscribed_events()

class TestEvent(TestCase):
    def test_get_event_name(self):
        self.assertEqual(Event.get_event_name(), 'Event')
        self.assertEqual(EventOne.get_event_name(), 'event_one')
        self.assertEqual(EventTwo.get_event_name(), 'event_two')
        self.assertEqual(EventThree.get_event_name(), 'event_three')

class TestEventDecorator(TestCase):
    def test_event_decorator(self):
        mediator = Mediator()
        event = stubs.VenusianEvent()
        mediator.dispatch(event)
        self.assertFalse(event.success)
        mediator.scan(package=stubs)
        mediator.dispatch(event)
        self.assertTrue(event.success)

class Mixin(object):
    def __init__(self):
        self.mixin_value = 'mixin-value'

class MediatorWithMixin(Mediator, Mixin):
    pass

class TestMultiInheritance(TestCase):
    def test_multi_inheritance(self):
        mediator = MediatorWithMixin()
        self.assertEqual(mediator.mixin_value, 'mixin-value')
<|endoftext|>


<commit_before>import unittest
import yql

from quote import get_yahoo_quote

class GetYahooQuoteTestCase(unittest.TestCase):
    """Test Case for the `get_yahoo_quote` function.

    The `get_yahoo_quote` function should query Yahoo's finance tables using YQL
    and return the latest quote for a particular stock (delayed by 20min).

    """
    def setUp(self):
        self.good_code = 'ABC'
        self.bad_code = 'A'

        self.columns = ['Name', 'Symbol', 'StockExchange', ]
        self.data = [u'ADEL BRTN FPO', u'ABC.AX', u'ASX', ]
        self.data_dict = dict(zip(self.columns, self.data))

    def test_good_code(self):
        """get_yahoo_quote should return True given a valid code."""
        ret, quote = get_yahoo_quote(self.good_code)
        self.assertTrue(ret)

    def test_bad_code(self):
        """get_yahoo_quote should raise an Exception given an invalid code."""
        self.assertRaises(Exception, get_yahoo_quote, self.bad_code)

    def test_get_columns(self):
        """get_yahoo_quote should return the requested column only."""
        ret, quote = get_yahoo_quote(self.good_code, self.columns)

        for key, value in self.data_dict.items():
            self.assertTrue(key in quote)
            self.assertEqual(quote[key], self.data_dict[key])

if __name__ == '__main__':
    unittest.main()

<commit_msg>Created unittests for a function that will validate a date range<commit_after>import unittest
import yql

from datetime import date
from decimal import Decimal

from quote import get_yahoo_quote, validate_date_range

class GetYahooQuoteTestCase(unittest.TestCase):
    """Test Case for the `get_yahoo_quote` function.

    The `get_yahoo_quote` function should query Yahoo's finance tables using YQL
    and return the latest quote for a particular stock (delayed by 20min).

    """
    def setUp(self):
        self.good_code = 'ABC'
        self.bad_code = 'A'

        self.columns = ['Name', 'Symbol', 'StockExchange', ]
        self.data = [u'ADEL BRTN FPO', u'ABC.AX', u'ASX', ]
        self.data_dict = dict(zip(self.columns, self.data))

    def test_good_code(self):
        """get_yahoo_quote should return True given a valid code."""
        ret, quote = get_yahoo_quote(self.good_code)
        self.assertTrue(ret)

    def test_bad_code(self):
        """get_yahoo_quote should raise an Exception given an invalid code."""
        self.assertRaises(Exception, get_yahoo_quote, self.bad_code)

    def test_get_columns(self):
        """get_yahoo_quote should return the requested column only."""
        ret, quote = get_yahoo_quote(self.good_code, self.columns)

        for key, value in self.data_dict.items():
            self.assertTrue(key in quote)
            self.assertEqual(quote[key], self.data_dict[key])

class ValidateDateRangeTestCase(unittest.TestCase):
    """Test Case for the `validate_date_range` function.

    The `validate_date_range` function will return True if the given date range
    validates against a set of custom rules.  Date ranges are used to extract
    historical data for a given stock.

    A valid date range is a list of two elements; each element may be a date
    object or a string representation of the date.

    """
    def setUp(self):
        # Some bad date range inputs
        self.bad_date_not_list = [2013, {}, '']
        self.bad_date_wrong_length = [
            [], ['2013-04-10'], ['2013-04-10', '2013-04-11', '2013-04-12', ]
        ]
        self.bad_date_wrong_types = [[2013, ''], [None, Decimal('2013')], ]
        self.bad_date_wrong_format = [date(2013, 4, 10), '12-04-2013']
        self.bad_date_backwards = ['2013-04-12', date(2013, 4, 10)]
        self.bad_date_future = [date(2020, 4, 10), '2020-04-12']

    def test_bad_date_not_list(self):
        """validate_date_range should raise a TypeError given a non-list date range."""
        [self.assertRaises(TypeError, validate_date_range, not_list)
            for not_list in self.bad_date_not_list]

    def test_bad_date_wrong_length(self):
        """validate_date_range should raise a ValueError given a list of incorrect length."""
        [self.assertRaises(ValueError, validate_date_range, wrong_length)
            for wrong_length in self.bad_date_wrong_length]

    def test_bad_date_wrong_type(self):
        """validate_date_range should raise a TypeError given a list with wrong types."""
        self.assertRaises(TypeError, validate_date_range, self.bad_date_wrong_types)

    def test_bad_date_wrong_format(self):
        """validate_date_range should raise a TypeError given a list with wrong format."""
        self.assertRaises(ValueError, validate_date_range, self.bad_date_wrong_format)

    def test_bad_date_backwards(self):
        """validate_date_range should raise an Exception given a backwards date range."""
        self.assertRaises(ValueError, validate_date_range, self.bad_date_backwards)

    def test_bad_date_future(self):
        """validate_date_range should raise an Exception given a future date range."""
        self.assertRaises(ValueError, validate_date_range, self.bad_date_future)

if __name__ == '__main__':
    unittest.main()

<|endoftext|>


<commit_before>import sys
import brainfuck
import unittest

_exec = brainfuck._exec
StringIO = brainfuck.StringIO

class ProgramTests(unittest.TestCase):
    def test_hello(self):
        self._test("programs/hello.bf", "", "Hello World!\n")

    def test_hello_small(self):
        self._test("programs/hellosmall.bf", "", "Hello World!\n")

    def test_rot13(self):
        self._test("programs/rot13.bf", "foobar", "sbbone")

    def _test(self, filename, input_str, expected_output):
        with open(filename, "r") as f:
            code = f.read()
        # Test function
        func = brainfuck.to_function(code)
        self.assertEqual(func(input_str), expected_output)
        # Test procedure
        func = brainfuck.to_procedure(code)
        out_, in_ = StringIO(), StringIO(input_str)
        func(out_, in_)
        self.assertEqual(out_.getvalue(), expected_output)
        # Test module
        module = brainfuck.to_module(code)
        out_, in_ = StringIO(), StringIO(input_str)
        old_out, old_in = sys.stdout, sys.stdin
        try:
            sys.stdout, sys.stdin = out_, in_
            _exec(module, globals(), locals())
        finally:
            sys.stdout, sys.stdin = old_out, old_in
        self.assertEqual(out_.getvalue(), expected_output)

class ImportHookTests(unittest.TestCase):
    def setUp(self):
        sys.path.append("programs")

    def tearDown(self):
        sys.path.remove("programs")
        brainfuck.remove_import_hook()
        # Remove imported brainfuck modules
        for key, module in list(sys.modules.items()):
            if hasattr(module, "__brainfuck_function__"):
                del sys.modules[key]

    def test_no_hook(self):
        with self.assertRaises(ImportError):
            import hello

    def test_install_hook(self):
        brainfuck.install_import_hook()
        import hello
        self.assertEqual(hello(), "Hello World!\n")

    def test_submodule(self):
        brainfuck.install_import_hook()
        import subdir.subhello
        self.assertEqual(subdir.subhello(), "Hello World!\n")

    def test_remove_hook(self):
        brainfuck.install_import_hook()
        brainfuck.remove_import_hook()
        with self.assertRaises(ImportError):
            import hello
        for importer in sys.meta_path:
            self.assertFalse(isinstance(importer, brainfuck.BrainfuckImporter))

    def test_custom_importer(self):
        cache = {"called": False}
        class Importer(brainfuck.BrainfuckImporter):
            def load_module(self, fullname):
                cache["called"] = True
                return super(Importer, self).load_module(fullname)

        brainfuck.install_import_hook(importer=Importer())
        import hello
        self.assertTrue(cache["called"])

    def test_custom_kwargs(self):
        brainfuck.install_import_hook(file_extensions=("bar",))
        for importer in sys.meta_path:
            if isinstance(importer, brainfuck.BrainfuckImporter):
                break
        self.assertEquals(importer.file_extensions, ("bar",))

if __name__ == "__main__":
    unittest.main()
<commit_msg>s/assertEquals/assertEqual/<commit_after>import sys
import brainfuck
import unittest

_exec = brainfuck._exec
StringIO = brainfuck.StringIO

class ProgramTests(unittest.TestCase):
    def test_hello(self):
        self._test("programs/hello.bf", "", "Hello World!\n")

    def test_hello_small(self):
        self._test("programs/hellosmall.bf", "", "Hello World!\n")

    def test_rot13(self):
        self._test("programs/rot13.bf", "foobar", "sbbone")

    def _test(self, filename, input_str, expected_output):
        with open(filename, "r") as f:
            code = f.read()
        # Test function
        func = brainfuck.to_function(code)
        self.assertEqual(func(input_str), expected_output)
        # Test procedure
        func = brainfuck.to_procedure(code)
        out_, in_ = StringIO(), StringIO(input_str)
        func(out_, in_)
        self.assertEqual(out_.getvalue(), expected_output)
        # Test module
        module = brainfuck.to_module(code)
        out_, in_ = StringIO(), StringIO(input_str)
        old_out, old_in = sys.stdout, sys.stdin
        try:
            sys.stdout, sys.stdin = out_, in_
            _exec(module, globals(), locals())
        finally:
            sys.stdout, sys.stdin = old_out, old_in
        self.assertEqual(out_.getvalue(), expected_output)

class ImportHookTests(unittest.TestCase):
    def setUp(self):
        sys.path.append("programs")

    def tearDown(self):
        sys.path.remove("programs")
        brainfuck.remove_import_hook()
        # Remove imported brainfuck modules
        for key, module in list(sys.modules.items()):
            if hasattr(module, "__brainfuck_function__"):
                del sys.modules[key]

    def test_no_hook(self):
        with self.assertRaises(ImportError):
            import hello

    def test_install_hook(self):
        brainfuck.install_import_hook()
        import hello
        self.assertEqual(hello(), "Hello World!\n")

    def test_submodule(self):
        brainfuck.install_import_hook()
        import subdir.subhello
        self.assertEqual(subdir.subhello(), "Hello World!\n")

    def test_remove_hook(self):
        brainfuck.install_import_hook()
        brainfuck.remove_import_hook()
        with self.assertRaises(ImportError):
            import hello
        for importer in sys.meta_path:
            self.assertFalse(isinstance(importer, brainfuck.BrainfuckImporter))

    def test_custom_importer(self):
        cache = {"called": False}
        class Importer(brainfuck.BrainfuckImporter):
            def load_module(self, fullname):
                cache["called"] = True
                return super(Importer, self).load_module(fullname)

        brainfuck.install_import_hook(importer=Importer())
        import hello
        self.assertTrue(cache["called"])

    def test_custom_kwargs(self):
        brainfuck.install_import_hook(file_extensions=("bar",))
        for importer in sys.meta_path:
            if isinstance(importer, brainfuck.BrainfuckImporter):
                break
        self.assertEqual(importer.file_extensions, ("bar",))

if __name__ == "__main__":
    unittest.main()
<|endoftext|>