<commit_before>import sys

from cps.base import BaseClient

class Client(BaseClient):

    def info(self, service_id):
        service = BaseClient.info(self, service_id)

        nodes = self.callmanager(service['sid'], "list_nodes", False, {})
        if 'error' in nodes:
            return

        print nodes

        for role in ( 'dir', 'mrc', 'osd' ):
            print "\n", role.upper(), "nodes:"

            for node in nodes[role]:
                params = { 'serviceNodeId': node }
                details = self.callmanager(service['sid'], "get_node_info", False, params)

                if 'error' in details:
                    print node, details['error']
                    continue

                if role == 'dir':
                    port = 30638

                if role == 'osd':
                    port = 30640

                if role == 'mrc':
                    port = 30636

                print "http://%s:%s" % (details['serviceNode']['ip'], port)

    def usage(self, cmdname):
        BaseClient.usage(self, cmdname)
        print "    add_nodes                  serviceid count    # add the specified number of osd nodes"
        print "    remove_nodes               serviceid count    # remove the specified number of osd nodes"
        print "    list_volumes               serviceid"
        print "    create_volume              serviceid vol_name"
        print "    delete_volume              serviceid vol_name"
        print "    list_osd_sel_policies      serviceid"
        print "    set_osd_sel_policy         serviceid vol_name policy"
        print "    list_replica_sel_policies  serviceid"
        print "    set_replica_sel_policy     serviceid vol_name policy"
        print "    list_replication_policies  serviceid"
        print "    set_replication_policy     serviceid vol_name policy factor"
# TODO: add when there is more than one striping policy
#        print "    list_striping_policies     serviceid"
#        print "    set_striping_policy        serviceid vol_name policy width stripe-size"

    def main(self, argv):
        command = argv[1]

        if command in ( 'add_nodes', 'remove_nodes', 'list_volumes', 'create_volume', 'delete_volume', 'list_striping_policies', 'list_replication_policies', 'list_osd_sel_policies', 'list_replica_sel_policies', 'set_striping_policy', 'set_replication_policy', 'set_osd_sel_policy', 'set_replica_sel_policy' ):
            try:
                sid = int(argv[2])
            except (IndexError, ValueError):
                self.usage(argv[0])
                sys.exit(0)

            self.check_service_id(sid)

        if command in ( 'add_nodes', 'remove_nodes' ):
            try:
                params = {
                    'osd': int(argv[3]),
                    #'dir': int(argv[4]),
                    #'mrc': int(argv[5])
                }
            except (IndexError, ValueError):
                self.usage(argv[0])
                sys.exit(0)

            if len(argv) == 4:
                params['cloud'] = 'default'
            else:
                params['cloud'] = argv[4]
            # call the method
            res = self.callmanager(sid, command, True, params)
            if 'error' in res:
                print res['error']
            else:
                print "Service", sid, "is performing the requested operation (%s)" % command

        if command == 'create_volume':
            try:
                params = { 'volumeName': argv[3] }
            except IndexError:
                self.usage(argv[0])
                sys.exit(0)

            res = self.callmanager(sid, 'createVolume', True, params)
            if 'error' in res:
                print res['error']
            else:
                print "Volume", params['volumeName'], "created"

        if command == 'delete_volume':
            try:
                params = { 'volumeName': argv[3] }
            except IndexError:
                self.usage(argv[0])
                sys.exit(0)

            res = self.callmanager(sid, 'deleteVolume', True, params)
            if 'error' in res:
                print res['error']
            else:
                print "Volume", params['volumeName'], "deleted"

        if command == 'list_volumes':
            res = self.callmanager(sid, 'listVolumes', False, {})
            if 'error' in res:
                print res['error']
            else:
                print res['volumes']

        if command in ( 'list_osd_sel_policies', 'list_replica_sel_policies', 'list_replication_policies', 'list_striping_policies' ):
            res = self.callmanager(sid, command, False, {})
            if 'error' in res:
                print res['error']
            else:
                print res['policies']

        if command in ( 'set_osd_sel_policy', 'set_replica_sel_policy' ):
            try:
                params = { 'volumeName': argv[3], 'policy': argv[4] }
            except IndexError:
                self.usage(argv[0])
                sys.exit(0)

            res = self.callmanager(sid, command, True, params)
            if 'error' in res:
                print res['error']
            else:
                print res['stdout']
                print "Policy set."

        if command in 'set_replication_policy':
            try:
                params = { 'volumeName': argv[3], 'policy': argv[4], 'factor': argv[5] }
            except IndexError:
                self.usage(argv[0])
                sys.exit(0)

            res = self.callmanager(sid, command, True, params)
            if 'error' in res:
                print res['error']
            else:
                print res['stdout']
                print "Policy set."

        if command in 'set_striping_policy':
            try:
                params = { 'volumeName': argv[3], 'policy': argv[4], 'width': argv[5], 'stripe-size': argv[6] }
            except IndexError:
                self.usage(argv[0])
                sys.exit(0)

            res = self.callmanager(sid, command, True, params)
            if 'error' in res:
                print res['error']
            else:
                print res['stdout']
                print "Policy set."

<commit_msg>conpaas xtreemfs service: replace individual policy-related commands with "list_policies" and "set_policy"<commit_after>import sys

from cps.base import BaseClient

class Client(BaseClient):

    def info(self, service_id):
        service = BaseClient.info(self, service_id)

        nodes = self.callmanager(service['sid'], "list_nodes", False, {})
        if 'error' in nodes:
            return

        print nodes

        for role in ( 'dir', 'mrc', 'osd' ):
            print "\n", role.upper(), "nodes:"

            for node in nodes[role]:
                params = { 'serviceNodeId': node }
                details = self.callmanager(service['sid'], "get_node_info", False, params)

                if 'error' in details:
                    print node, details['error']
                    continue

                if role == 'dir':
                    port = 30638

                if role == 'osd':
                    port = 30640

                if role == 'mrc':
                    port = 30636

                print "http://%s:%s" % (details['serviceNode']['ip'], port)

    def usage(self, cmdname):
        BaseClient.usage(self, cmdname)
        print "    add_nodes         serviceid count [cloud] # add the specified number of osd nodes"
        print "    remove_nodes      serviceid count [cloud] # remove the specified number of osd nodes"
        print "    list_volumes      serviceid"
        print "    create_volume     serviceid vol_name"
        print "    delete_volume     serviceid vol_name"
        print "    list_policies     serviceid policy_type # [ osd_sel | replica_sel | replication ]"
        print "    set_policy        serviceid policy_type vol_name policy [factor]"
# TODO: add when there is more than one striping policy
#        print "    list_striping_policies     serviceid"
#        print "    set_striping_policy        serviceid vol_name policy width stripe-size"

    def main(self, argv):
        command = argv[1]

        if command in ( 'add_nodes', 'remove_nodes', 'list_volumes',
                        'create_volume', 'delete_volume',
                        'list_policies', 'set_policy' ):
            try:
                sid = int(argv[2])
            except (IndexError, ValueError):
                self.usage(argv[0])
                sys.exit(0)

            self.check_service_id(sid)

        if command in ( 'add_nodes', 'remove_nodes' ):
            try:
                params = {
                    'osd': int(argv[3]),
                    #'dir': int(argv[4]),
                    #'mrc': int(argv[5])
                }
            except (IndexError, ValueError):
                self.usage(argv[0])
                sys.exit(0)

            if len(argv) == 4:
                params['cloud'] = 'default'
            else:
                params['cloud'] = argv[4]
            # call the method
            res = self.callmanager(sid, command, True, params)
            if 'error' in res:
                print res['error']
            else:
                print "Service", sid, "is performing the requested operation (%s)" % command

        if command == 'create_volume':
            try:
                params = { 'volumeName': argv[3] }
            except IndexError:
                self.usage(argv[0])
                sys.exit(0)

            res = self.callmanager(sid, 'createVolume', True, params)
            if 'error' in res:
                print res['error']
            else:
                print "Volume", params['volumeName'], "created"

        if command == 'delete_volume':
            try:
                params = { 'volumeName': argv[3] }
            except IndexError:
                self.usage(argv[0])
                sys.exit(0)

            res = self.callmanager(sid, 'deleteVolume', True, params)
            if 'error' in res:
                print res['error']
            else:
                print "Volume", params['volumeName'], "deleted"

        if command == 'list_volumes':
            res = self.callmanager(sid, 'listVolumes', False, {})
            if 'error' in res:
                print res['error']
            else:
                print res['volumes']

        if command == 'list_policies':
            try:
                policy_type = argv[3]
            except IndexError:
                self.usage(argv[0])
                sys.exit(0)

            if policy_type == 'osd_sel':
                command = 'list_osd_sel_policies'

            elif policy_type == 'replica_sel':
                command = 'list_replica_sel_policies'

            elif policy_type == 'replication':
                command = 'list_replication_policies'

            else:
                self.usage(argv[0])
                sys.exit(0)

            res = self.callmanager(sid, command, False, {})
            if 'error' in res:
                print res['error']
            else:
                print res['policies']

        if command == 'set_policy':
            try:
                policy_type = argv[3]
            except IndexError:
                self.usage(argv[0])
                sys.exit(0)

            try:
                params = { 'volumeName': argv[4], 'policy': argv[5] }
            except IndexError:
                self.usage(argv[0])
                sys.exit(0)

            if policy_type == 'osd_sel':
                command = 'set_osd_sel_policy'

            elif policy_type == 'replica_sel':
                command = 'set_replica_sel_policy'

            elif policy_type == 'replication':
                command = 'set_replication_policy'
                try:
                    # set_replication_policy requires a 'factor'
                    params['factor'] = argv[6]
                except IndexError:
                    self.usage(argv[0])
                    sys.exit(0)
            else:
                self.usage(argv[0])
                sys.exit(0)

            res = self.callmanager(sid, command, True, params)
            if 'error' in res:
                print res['error']
            else:
                print res['stdout']
                print "Policy set."

#        if command in 'set_striping_policy':
#            try:
#                params = { 'volumeName': argv[3], 'policy': argv[4], 'width': argv[5], 'stripe-size': argv[6] }
#            except IndexError:
#                self.usage(argv[0])
#                sys.exit(0)
#
#            res = self.callmanager(sid, command, True, params)
#            if 'error' in res:
#                print res['error']
#            else:
#                print res['stdout']
#                print "Policy set."

<|endoftext|>


<commit_before># TODO:
#   - display inactive record tags
#   - add CSV output
#   - add custom template support

import sys
import logging
import calendar

from jinja2 import Environment
import argparse

from BinaryParser import Mmap
from MFT import Cache
from MFT import MFTEnumerator
from get_file_info import make_model

def unixtimestampformat(value):
    if value is None:
        return 0
    return int(calendar.timegm(value.timetuple()))

def get_default_template(env):
    return env.from_string(
"""\
{% if record.standard_information and record.filename_information %}\
0|{{ record.path }}|{{ record.record_num }}|0|{{ record.standard_information.owner_id }}|0|{{ record.size }}|{{ record.standard_information.accessed|unixtimestampformat }}|{{ record.standard_information.modified|unixtimestampformat }}|{{ record.standard_information.changed|unixtimestampformat }}|{{ record.standard_information.created|unixtimestampformat }}
{% endif %}\
{% if record.standard_information and record.filename_information %}\
0|{{ record.path }} (filename)|{{ record.record_num }}|0|{{ record.standard_information.owner_id }}|0|{{ record.size }}|{{ record.filename_information.accessed|unixtimestampformat }}|{{ record.filename_information.modified|unixtimestampformat }}|{{ record.filename_information.changed|unixtimestampformat }}|{{ record.filename_information.created|unixtimestampformat }}
{% endif %}\
{% for e in record.indx_entries %}\
0|{{ record.path }}\\{{ e.name }} (INDX)|{{ e.record_num }}|0|0|0|{{ e.logical_size }}|{{ e.accessed|unixtimestampformat }}|{{ e.modified|unixtimestampformat }}|{{ e.changed|unixtimestampformat }}|{{ e.created|unixtimestampformat }}
{% endfor %}\
{% for e in record.slack_indx_entries %}\
0|{{ record.path }}\\{{ e.name }} (slack-INDX)|{{ e.record_num }}|0|0|0|{{ e.logical_size }}|{{ e.accessed|unixtimestampformat }}|{{ e.modified|unixtimestampformat }}|{{ e.changed|unixtimestampformat }}|{{ e.created|unixtimestampformat }}
{% endfor %}
""")

def main():
    parser = argparse.ArgumentParser(description='Parse MFT '
                                     'filesystem structures.')
    parser.add_argument('-f', action="store", metavar="regex",
                        nargs=1, dest="filter",
                        help="Only consider entries whose path "
                        "matches this regular expression")
    parser.add_argument('-c', action="store", metavar="cache_size", type=int,
                        dest="cache_size", default=1024,
                        help="Size of cache.")
    parser.add_argument('-p', action="store", metavar="prefix",
                        nargs=1, dest="prefix", default="\\.",
                        help="Prefix paths with `prefix` rather than \\.\\")
    parser.add_argument('--progress', action="store_true",
                        dest="progress",
                        help="Update a status indicator on STDERR "
                        "if STDOUT is redirected")
    parser.add_argument('-v', action="store_true", dest="verbose",
                        help="Print debugging information")
    parser.add_argument('filename', action="store",
                        help="Input MFT file path")

    results = parser.parse_args()

    if results.verbose:
        logging.basicConfig(level=logging.DEBUG)

    env = Environment(trim_blocks=True, lstrip_blocks=True)
    env.filters["unixtimestampformat"] = unixtimestampformat

    template = get_default_template(env)
    with Mmap(results.filename) as buf:
        record_cache = Cache(results.cache_size)
        path_cache = Cache(results.cache_size)

        enum = MFTEnumerator(buf,
                             record_cache=record_cache,
                             path_cache=path_cache)
        for record, record_path in enum.enumerate_paths():
            sys.stdout.write(template.render(record=make_model(record, record_path)))

if __name__ == "__main__":
    main()
<commit_msg>add progress bar back to list-mft<commit_after># TODO:
#   - display inactive record tags
#   - add CSV output
#   - add custom template support

import types
import sys
import logging
import calendar
import json
import datetime

from jinja2 import Environment
import argparse

from BinaryParser import Mmap
from MFT import Cache
from MFT import MFTEnumerator
from get_file_info import make_model
from Progress import NullProgress
from Progress import ProgressBarProgress

def unixtimestampformat(value):
    if value is None:
        return 0
    return int(calendar.timegm(value.timetuple()))

def get_default_template(env):
    return env.from_string(
"""\
{% if record.standard_information and record.filename_information %}
0|{{ prefix }}{{ record.path }}|{{ record.inode }}|0|{{ record.standard_information.owner_id }}|0|{{ record.size }}|{{ record.standard_information.accessed|unixtimestampformat }}|{{ record.standard_information.modified|unixtimestampformat }}|{{ record.standard_information.changed|unixtimestampformat }}|{{ record.standard_information.created|unixtimestampformat }}
{% endif %}
{% if record.standard_information and record.filename_information %}
0|{{ prefix }}{{ record.path }} (filename)|{{ record.inode }}|0|{{ record.standard_information.owner_id }}|0|{{ record.size }}|{{ record.filename_information.accessed|unixtimestampformat }}|{{ record.filename_information.modified|unixtimestampformat }}|{{ record.filename_information.changed|unixtimestampformat }}|{{ record.filename_information.created|unixtimestampformat }}
{% endif %}
{% for e in record.indx_entries %}
0|{{ prefix }}{{ record.path }}\\{{ e.name }} (INDX)|{{ e.inode }}|0|0|0|{{ e.logical_size }}|{{ e.accessed|unixtimestampformat }}|{{ e.modified|unixtimestampformat }}|{{ e.changed|unixtimestampformat }}|{{ e.created|unixtimestampformat }}
{% endfor %}
{% for e in record.slack_indx_entries %}
0|{{ prefix }}{{ record.path }}\\{{ e.name }} (slack-INDX)|{{ e.inode }}|0|0|0|{{ e.logical_size }}|{{ e.accessed|unixtimestampformat }}|{{ e.modified|unixtimestampformat }}|{{ e.changed|unixtimestampformat }}|{{ e.created|unixtimestampformat }}
{% endfor %}
""")

def main():
    parser = argparse.ArgumentParser(description='Parse MFT '
                                     'filesystem structures.')
    parser.add_argument('-c', action="store", metavar="cache_size", type=int,
                        dest="cache_size", default=1024,
                        help="Size of cache.")
    parser.add_argument('-p', action="store", metavar="prefix",
                        nargs=1, dest="prefix", default="\\.",
                        help="Prefix paths with `prefix` rather than \\.\\")
    parser.add_argument('-v', action="store_true", dest="verbose",
                        help="Print debugging information")
    parser.add_argument('--progress', action="store_true",
                        dest="progress",
                        help="Update a status indicator on STDERR "
                        "if STDOUT is redirected")
    parser.add_argument('--format', action="store", metavar="format",
                        nargs=1, dest="format",
                        help="Output format specification")
    parser.add_argument('--format_file', action="store", metavar="format_file",
                        nargs=1, dest="format_file",
                        help="File containing output format specification")
    parser.add_argument('--json', action="store_true", dest="json",
                        help="Output in JSON format")
    parser.add_argument('-f', action="store", metavar="regex",
                        nargs=1, dest="filter",
                        help="Only consider entries whose path "
                        "matches this regular expression")
    parser.add_argument('filename', action="store",
                        help="Input MFT file path")
    results = parser.parse_args()

    if results.verbose:
        logging.basicConfig(level=logging.DEBUG)

    env = Environment(trim_blocks=True, lstrip_blocks=True)
    env.filters["unixtimestampformat"] = unixtimestampformat

    flags_count = 0
    if results.format:
        flags_count += 1
        template = env.from_string(results.format[0])
    if results.format_file:
        flags_count += 1
        with open(results.format_file[0], "rb") as f:
            template = env.from_string(f.read())
    if results.json:
        flags_count += 1
        pass

    if flags_count > 1:
        sys.stderr.write("Only one of --format, --format_file, --json may be provided.\n")
        sys.exit(-1)
    elif flags_count == 0:
        flags_count += 1
        template = get_default_template(env)

    if results.progress:
        progress_cls = ProgressBarProgress
    else:
        progress_cls = NullProgress

    with Mmap(results.filename) as buf:
        record_cache = Cache(results.cache_size)
        path_cache = Cache(results.cache_size)

        enum = MFTEnumerator(buf,
                             record_cache=record_cache,
                             path_cache=path_cache)
        progress = progress_cls(enum.len())
        if results.json:
            class MFTEncoder(json.JSONEncoder):
                def default(self, obj):
                    if isinstance(obj, datetime.datetime):
                        return obj.isoformat("T") + "Z"
                    elif isinstance(obj, types.GeneratorType):
                        return [o for o in obj]
                    return json.JSONEncoder.default(self, obj)
            print("[")
            for record, record_path in enum.enumerate_paths():
                m = make_model(record, record_path)
                print(json.dumps(m, cls=MFTEncoder, indent=2) + ",")
                progress.set_current(record.inode)
            print("]")
        else:
            for record, record_path in enum.enumerate_paths():
                sys.stdout.write(template.render(record=make_model(record, record_path),
                                                 prefix=results.prefix[0]))
                progress.set_current(record.inode)
        progress.set_complete()

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before>import asyncio
import aiomysql

loop = asyncio.get_event_loop()

@asyncio.coroutine
def test_example_transaction():
    conn = yield from aiomysql.connect(host='127.0.0.1', port=3306,
                                       user='root', password='',
                                       db='test_pymysql', autocommit=False,
                                       loop=loop)

    cursor = yield from conn.cursor()
    stmt_drop = "DROP TABLE IF EXISTS names"
    yield from cursor.execute(stmt_drop)
    yield from cursor.execute("""
        CREATE TABLE names (
        id TINYINT UNSIGNED NOT NULL AUTO_INCREMENT,
        name VARCHAR(30) DEFAULT '' NOT NULL,
        cnt TINYINT UNSIGNED DEFAULT 0,
        PRIMARY KEY (id))""")
    yield from conn.commit()

    # Insert 3 records
    names = (('Geert',), ('Jan',), ('Michel',))
    stmt_insert = "INSERT INTO names (name) VALUES (%s)"
    yield from cursor.executemany(stmt_insert, names)

    # Roll back!!!!
    yield from conn.rollback()

    # There should be no data!
    stmt_select = "SELECT id, name FROM names ORDER BY id"
    yield from cursor.execute(stmt_select)
    resp = yield from cursor.fetchall()
    # Check there is no data
    assert not resp

    # Do the insert again.
    yield from cursor.executemany(stmt_insert, names)

    # Data should be already there
    yield from cursor.execute(stmt_select)
    resp = yield from cursor.fetchall()
    print(resp)
    # Do a commit
    conn.commit()

    yield from cursor.execute(stmt_select)
    print(resp)

    # Cleaning up, dropping the table again
    yield from cursor.execute(stmt_drop)
    yield from cursor.close()
    conn.close()

loop.run_until_complete(test_example_transaction())
<commit_msg>add missed yield from in example<commit_after>import asyncio
import aiomysql

loop = asyncio.get_event_loop()

@asyncio.coroutine
def test_example_transaction():
    conn = yield from aiomysql.connect(host='127.0.0.1', port=3306,
                                       user='root', password='',
                                       db='test_pymysql', autocommit=False,
                                       loop=loop)

    cursor = yield from conn.cursor()
    stmt_drop = "DROP TABLE IF EXISTS names"
    yield from cursor.execute(stmt_drop)
    yield from cursor.execute("""
        CREATE TABLE names (
        id TINYINT UNSIGNED NOT NULL AUTO_INCREMENT,
        name VARCHAR(30) DEFAULT '' NOT NULL,
        cnt TINYINT UNSIGNED DEFAULT 0,
        PRIMARY KEY (id))""")
    yield from conn.commit()

    # Insert 3 records
    names = (('Geert',), ('Jan',), ('Michel',))
    stmt_insert = "INSERT INTO names (name) VALUES (%s)"
    yield from cursor.executemany(stmt_insert, names)

    # Roll back!!!!
    yield from conn.rollback()

    # There should be no data!
    stmt_select = "SELECT id, name FROM names ORDER BY id"
    yield from cursor.execute(stmt_select)
    resp = yield from cursor.fetchall()
    # Check there is no data
    assert not resp

    # Do the insert again.
    yield from cursor.executemany(stmt_insert, names)

    # Data should be already there
    yield from cursor.execute(stmt_select)
    resp = yield from cursor.fetchall()
    print(resp)
    # Do a commit
    yield from conn.commit()

    yield from cursor.execute(stmt_select)
    print(resp)

    # Cleaning up, dropping the table again
    yield from cursor.execute(stmt_drop)
    yield from cursor.close()
    conn.close()

loop.run_until_complete(test_example_transaction())
<|endoftext|>


<commit_before>#!/usr/bin/env python3

"""diffout

Usage:
  diffout [options] [--] <commandline> <infile>...
  diffout -h | --help
  diffout --version

Runs a command on a list of input files, then compares the resulting outputs with copies stored from a previous run.

Examples:
  diffout <commandline> <infile>...

Options:
  -s, --save       Clear expected results, then save all test generated output as new expected results.
  -p, --pipe       In addition to normal output files, diff terminal output as well
  -h, --help       Show help.
  -q, --quiet      Print less text.
  -v, --verbose    Print more text.
  --version        Show version.
"""

from docopt import docopt
import logging
import difflib
import glob
import re
import os
import shutil
import shlex
import subprocess
import time
#import colorama

VERSION="0.1.0" # MAJOR.MINOR.PATCH | http://semver.org

HTML_PATH         = os.path.join("diffout","diffs")
EXPECTED_PATH     = os.path.join("diffout","expected")
OUTPUT_PATH       = os.path.join("diffout","output")
TERMINAL_OUT_PATH = os.path.join("diffout","output")

def fatal( errorMsg ):
	logging.critical(errorMsg)
	exit(1)
	return

def loadFile(fn):
	inBuf = []
	encoding = ""

	if not os.path.isfile(fn):
		fatal("File not found: {}".format(fn))

	if encoding == "":
		try:
			wbuf = open(fn, "r", encoding='ascii').read()
			encoding = "ASCII"
			inBuf = wbuf.split("\n")
		except Exception as e:
			pass

	if encoding == "":
		try:
			wbuf = open(fn, "rU", encoding='UTF-8').read()
			encoding = "utf_8"
			inBuf = wbuf.split("\n")
			# remove BOM on first line if present
			t = ":".join("{0:x}".format(ord(c)) for c in inBuf[0])
			if t[0:4] == 'feff':
				inBuf[0] = inBuf[0][1:]
		except:
			pass

	if encoding == "":
		try:
			wbuf = open(fn, "r", encoding='latin_1').read()
			encoding = "latin_1"
			inBuf = wbuf.split("\n")
		except Exception as e:
			pass

	if encoding == "":
		fatal("Cannot determine input file decoding")

	return inBuf;

def expandPath( path ):
	path = os.path.abspath(path)
	path = os.path.expanduser(path)
	path = os.path.expandvars(path)
	path = os.path.normpath(path)

	return path

def getFilesModifiedAfterFile( path ):

	path = expandPath(path)
	startTime = os.path.getmtime(path)

	lp, rp = os.path.split(path)
	searchPath = os.path.join(lp, "*")

	modifiedFiles = []
	for f in glob.glob(searchPath):
		mtime = os.path.getmtime(f)
		if mtime > startTime:
			modifiedFiles.append(f)

	return modifiedFiles

def getDirectoryFileList( path ):
	path = expandPath(path)
	if not os.path.isdir(path):
		logging.error("{} is not a directory".format(path))
		return []

	searchPath = os.path.join(path,"*")
	return glob.glob(searchPath)

def saveFiles( fileList, destDir ):
	destDir = expandPath(destDir)
	for f in fileList:
		f = expandPath(f)
		lp, rp = os.path.split(f)

		if not os.path.exists(destDir):
			os.makedirs(destDir)

		dstFile = os.path.join(destDir,rp)
		srcFile = f
		shutil.copy(srcFile,dstFile)

def diffDir( newDir, oldDir ):
	logging.info("--- Comparing new outputs with expected outputs:")

	newDir = expandPath(newDir)
	oldDir = expandPath(oldDir)
	newFiles = getDirectoryFileList(newDir)
	oldFiles = getDirectoryFileList(oldDir)

	if not os.path.exists(HTML_PATH):
		os.makedirs(HTML_PATH)

	# Index HTML Header
	indexHtml = []
	indexHtml.extend(htmlHeader)
	indexHtml.append("
")

	#for f in sorted(extraFiles):
		#logging.info("Unexpected output file was generated: {}".format(f))

	fileChangeCount = 0
	d = difflib.HtmlDiff(8,80)
	for f in sorted(newFiles):
		# HTML Header
		outBuf = []
		outBuf.extend(htmlHeader)

		# Diffs
		fn = os.path.basename(f)
		expectedFilePath = os.path.join(oldDir,fn)

		actual = loadFile(f)
		diffResult = ""
		if os.path.exists(expectedFilePath):
			expected = loadFile(expectedFilePath)

			#matchText = colorama.Style.BRIGHT + colorama.Back.LIGHTRED_EX + "[ DIFF   ]" + colorama.Back.RESET + colorama.Style.RESET_ALL
			matchText = "[ DIFF   ]"
			if actual==expected:
				diffResult = "NODIFF"
				#matchText = colorama.Back.LIGHTGREEN_EX + "[ NODIFF ]" + colorama.Style.RESET_ALL
				matchText = "[ NODIFF ]"
			else:
				diffResult = "DIFF"
				fileChangeCount += 1

			print("{} {}".format(matchText,f))

			s = d.make_table(expected, actual, expectedFilePath, f, True)
			outBuf.append(s)
			outBuf.append('
')
		else:
			diffResult = "EXTRA"
			#matchText = colorama.Style.BRIGHT + colorama.Back.BLUE + "[ EXTRA  ]" + colorama.Back.RESET + colorama.Style.RESET_ALL
			#print("{} Unexpected output file was generated: {}".format(matchText,os.path.basename(f)))

		# HTML Footer
		outBuf.extend(htmlFooter)

		# Write out results
		p = os.path.join(HTML_PATH,"{}.html".format(fn))
		htmlout = open(p, mode='w', encoding="utf-8")
		htmlout.writelines(["%s\n" % item for item in outBuf])
		htmlout.close()

		# Add to index
		if diffResult == "DIFF":
			resultCell = "DIFF	"
		elif diffResult == "NODIFF":
			resultCell = "NODIFF	"
		elif diffResult == "EXTRA":
			resultCell = "EXTRA	"

		indexHtml.append("
{4}{0}	diff results	output file	expected output file
".format(fn,os.path.basename(HTML_PATH),os.path.basename(OUTPUT_PATH),os.path.basename(EXPECTED_PATH),resultCell))

	# Check for missing/unexpected files
	nfSet = set()
	for f in newFiles:
		nfSet.add(os.path.basename(f))
	ofSet = set()
	for f in oldFiles:
		ofSet.add(os.path.basename(f))

	extraFiles = nfSet - ofSet
	missingFiles = ofSet - nfSet
	for f in sorted(missingFiles):
		#matchText = colorama.Style.BRIGHT + colorama.Back.CYAN + "[ MISSING]" + colorama.Back.RESET + colorama.Style.RESET_ALL
		#print("{} Expected output file not generated: {}".format(matchText,f))
		indexHtml.append("
MISSING	{0}	diff results	output file	expected output file
".format(fn,os.path.basename(HTML_PATH),os.path.basename(OUTPUT_PATH),os.path.basename(EXPECTED_PATH)))

	# Index HTML Footer
	indexHtml.append("
")
	indexHtml.append("")
	indexHtml.append("")

	# Write out index.html
	parentDir = os.path.dirname(HTML_PATH)
	p = os.path.join(parentDir,"results.html")
	htmlout = open(p, mode='w', encoding="utf-8")
	htmlout.writelines(["%s\n" % item for item in indexHtml])
	htmlout.close()

	# Finished, summarize results
	print()
	if len(extraFiles) > 0:
		print("{} unexpected output files were generated.".format(len(extraFiles)))
	if len(missingFiles) > 0:
		print("{} expected output files were not generated.".format(len(missingFiles)))
	if fileChangeCount > 0:
		print("{} output file(s) differ with expected output, view diffout/results.html for diff results".format(fileChangeCount))
	else:
		print("No differences with expected output found.")
	print()

	return

def main():
	testDirectoryPath = "~/pp/tools/repo/diffout/"
	args = docopt(__doc__, version="diffout v{}".format(VERSION))

	#colorama.init()

	# Configure logging
	logLevel = logging.INFO #default
	if args['--verbose']:
		logLevel = logging.DEBUG
	elif args['--quiet']:
		logLevel = logging.ERROR

	logging.basicConfig(format='%(levelname)s: %(message)s', level=logLevel)
	logging.debug(args)

	# Make directories if missing
	if not os.path.exists(EXPECTED_PATH):
		os.makedirs(EXPECTED_PATH)
	if not os.path.exists(OUTPUT_PATH):
		os.makedirs(OUTPUT_PATH)
	if not os.path.exists(HTML_PATH):
		os.makedirs(HTML_PATH)
	if not os.path.exists(TERMINAL_OUT_PATH):
		os.makedirs(TERMINAL_OUT_PATH)

	# Save command
	if args['--save']:
		p = expandPath(EXPECTED_PATH)
		if os.path.exists(p):
			logging.info("--- Clearing current expected results")
			print(p)
			shutil.rmtree(p)
			os.makedirs(p)

		logging.info("--- Saving latest results to expected")
		saveFiles(getDirectoryFileList(OUTPUT_PATH),EXPECTED_PATH)
		return

	# Delete output files from previous run
	p = expandPath(OUTPUT_PATH)
	if os.path.exists(p):
		logging.info("--- Clearing results from last run")
		print(p)
		shutil.rmtree(p)
		os.makedirs(p)

	# Delete diff from previous run
	p = expandPath(HTML_PATH)
	if os.path.exists(p):
		logging.info("--- Clearing diffs from last run")
		print(p)
		shutil.rmtree(p)
		os.makedirs(p)

	# Delete terminal output files from previous run
	p = expandPath(TERMINAL_OUT_PATH)
	if os.path.exists(p):
		logging.info("--- Clearing terminal output logs from last run")
		print(p)
		shutil.rmtree(p)
		os.makedirs(p)

	# Write marker file for time index
	f = open("STARTTIME",'w')
	f.write('.')
	f.close()
	time.sleep(1)

	# Run command on each input file
	commandCount = 0
	commandErrorCount = 0
	for infile in args['<infile>']:
		for f in glob.glob(infile):
			commandline = args['<commandline>']
			commandline = commandline.replace('%F',f)
			terminalOutFile = open(os.path.join(TERMINAL_OUT_PATH,os.path.basename(f))+".out",'w')

			#s = colorama.Fore.LIGHTYELLOW_EX + "\n----- Running command:\n{}\n".format(commandline) + colorama.Fore.RESET
			s = "\n----- Running command:\n{}\n".format(commandline)
			print(s)
			commandCount += 1

			with terminalOutFile as outFile:
				cl = shlex.split(commandline)
				logging.debug("args: {}".format(str(cl)))
				if args['--pipe']:
					proc=subprocess.Popen(cl, stdout=outFile, stderr=outFile)
				else:
					proc=subprocess.Popen(cl)
				proc.wait()
				if( proc.returncode != 0 ):
					logging.error("Command failed: {}".format(commandline))
					commandErrorCount += 1

	# Copy recently modified files into output/
	outFiles = getFilesModifiedAfterFile("STARTTIME")
	saveFiles(outFiles,OUTPUT_PATH)

	print("\nFinished executing {} command(s) ({} error(s) occured).".format(commandCount,commandErrorCount))
	print("{} output file(s) were generated ({} expected).".format(len(getDirectoryFileList(OUTPUT_PATH)),len(getDirectoryFileList(EXPECTED_PATH))))
	print()

	diffDir(OUTPUT_PATH,EXPECTED_PATH)

	return

htmlHeader = ('<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">',
			  '',
			  '',
			  '',
			  ' <meta http-equiv="Content-Type" content="text/html; charset=UTF-8" />',
			  ' <title>diffout results</title>',
			  ' ',
			  '',
			  '',
			  '')

htmlFooter = ('
',
             '
  Legends
',
             '

',
             '
 Colors
',
             '
 Added
',
             '
Changed
',
             '
Deleted
',
             '
	',
             '
',
             '
 Links
',
             '
(f)irst change
',
             '
(n)ext change
',
             '
(t)op
',
             '
',
             '
',
             '
',
             '',
             '',
             '')

if __name__ == "__main__":
	main()
<commit_msg>Display correct filename on MISSING files<commit_after>#!/usr/bin/env python3

"""diffout

Usage:
  diffout [options] [--] <commandline> <infile>...
  diffout -s
  diffout -h | --help
  diffout --version

Runs a command on a list of input files, then compares the resulting outputs with copies stored from a previous run.

Examples:
  diffout <commandline> <infile>...

Options:
  -s, --save       Clear expected results, then save all test generated output as new expected results.
  -p, --pipe       In addition to normal output files, diff terminal output as well
  -h, --help       Show help.
  -q, --quiet      Print less text.
  -v, --verbose    Print more text.
  --version        Show version.
"""

from docopt import docopt
import logging
import difflib
import glob
import re
import os
import shutil
import shlex
import subprocess
import time
#import colorama

VERSION="0.1.0" # MAJOR.MINOR.PATCH | http://semver.org

HTML_PATH         = os.path.join("diffout","diffs")
EXPECTED_PATH     = os.path.join("diffout","expected")
OUTPUT_PATH       = os.path.join("diffout","output")
TERMINAL_OUT_PATH = os.path.join("diffout","output")

def fatal( errorMsg ):
	logging.critical(errorMsg)
	exit(1)
	return

def loadFile(fn):
	inBuf = []
	encoding = ""

	if not os.path.isfile(fn):
		fatal("File not found: {}".format(fn))

	if encoding == "":
		try:
			wbuf = open(fn, "r", encoding='ascii').read()
			encoding = "ASCII"
			inBuf = wbuf.split("\n")
		except Exception as e:
			pass

	if encoding == "":
		try:
			wbuf = open(fn, "rU", encoding='UTF-8').read()
			encoding = "utf_8"
			inBuf = wbuf.split("\n")
			# remove BOM on first line if present
			t = ":".join("{0:x}".format(ord(c)) for c in inBuf[0])
			if t[0:4] == 'feff':
				inBuf[0] = inBuf[0][1:]
		except:
			pass

	if encoding == "":
		try:
			wbuf = open(fn, "r", encoding='latin_1').read()
			encoding = "latin_1"
			inBuf = wbuf.split("\n")
		except Exception as e:
			pass

	if encoding == "":
		fatal("Cannot determine input file decoding")

	return inBuf;

def expandPath( path ):
	path = os.path.abspath(path)
	path = os.path.expanduser(path)
	path = os.path.expandvars(path)
	path = os.path.normpath(path)

	return path

def getFilesModifiedAfterFile( path ):

	path = expandPath(path)
	startTime = os.path.getmtime(path)

	lp, rp = os.path.split(path)
	searchPath = os.path.join(lp, "*")

	modifiedFiles = []
	for f in glob.glob(searchPath):
		mtime = os.path.getmtime(f)
		if mtime > startTime:
			modifiedFiles.append(f)

	return modifiedFiles

def getDirectoryFileList( path ):
	path = expandPath(path)
	if not os.path.isdir(path):
		logging.error("{} is not a directory".format(path))
		return []

	searchPath = os.path.join(path,"*")
	return glob.glob(searchPath)

def saveFiles( fileList, destDir ):
	destDir = expandPath(destDir)
	for f in fileList:
		f = expandPath(f)
		lp, rp = os.path.split(f)

		if not os.path.exists(destDir):
			os.makedirs(destDir)

		dstFile = os.path.join(destDir,rp)
		srcFile = f
		shutil.copy(srcFile,dstFile)

def diffDir( newDir, oldDir ):
	logging.info("--- Comparing new outputs with expected outputs:")

	newDir = expandPath(newDir)
	oldDir = expandPath(oldDir)
	newFiles = getDirectoryFileList(newDir)
	oldFiles = getDirectoryFileList(oldDir)

	if not os.path.exists(HTML_PATH):
		os.makedirs(HTML_PATH)

	# Index HTML Header
	indexHtml = []
	indexHtml.extend(htmlHeader)
	indexHtml.append("
")

	#for f in sorted(extraFiles):
		#logging.info("Unexpected output file was generated: {}".format(f))

	fileChangeCount = 0
	d = difflib.HtmlDiff(8,80)
	for f in sorted(newFiles):
		# HTML Header
		outBuf = []
		outBuf.extend(htmlHeader)

		# Diffs
		fn = os.path.basename(f)
		expectedFilePath = os.path.join(oldDir,fn)

		actual = loadFile(f)
		diffResult = ""
		if os.path.exists(expectedFilePath):
			expected = loadFile(expectedFilePath)

			#matchText = colorama.Style.BRIGHT + colorama.Back.LIGHTRED_EX + "[ DIFF   ]" + colorama.Back.RESET + colorama.Style.RESET_ALL
			matchText = "[ DIFF   ]"
			if actual==expected:
				diffResult = "NODIFF"
				#matchText = colorama.Back.LIGHTGREEN_EX + "[ NODIFF ]" + colorama.Style.RESET_ALL
				matchText = "[ NODIFF ]"
			else:
				diffResult = "DIFF"
				fileChangeCount += 1

			print("{} {}".format(matchText,f))

			s = d.make_table(expected, actual, expectedFilePath, f, True)
			outBuf.append(s)
			outBuf.append('
')
		else:
			diffResult = "EXTRA"
			#matchText = colorama.Style.BRIGHT + colorama.Back.BLUE + "[ EXTRA  ]" + colorama.Back.RESET + colorama.Style.RESET_ALL
			#print("{} Unexpected output file was generated: {}".format(matchText,os.path.basename(f)))

		# HTML Footer
		outBuf.extend(htmlFooter)

		# Write out results
		p = os.path.join(HTML_PATH,"{}.html".format(fn))
		htmlout = open(p, mode='w', encoding="utf-8")
		htmlout.writelines(["%s\n" % item for item in outBuf])
		htmlout.close()

		# Add to index
		if diffResult == "DIFF":
			resultCell = "DIFF	"
		elif diffResult == "NODIFF":
			resultCell = "NODIFF	"
		elif diffResult == "EXTRA":
			resultCell = "EXTRA	"

		indexHtml.append("
{4}{0}	diff results	output file	expected output file
".format(fn,os.path.basename(HTML_PATH),os.path.basename(OUTPUT_PATH),os.path.basename(EXPECTED_PATH),resultCell))

	# Check for missing/unexpected files
	nfSet = set()
	for f in newFiles:
		nfSet.add(os.path.basename(f))
	ofSet = set()
	for f in oldFiles:
		ofSet.add(os.path.basename(f))

	extraFiles = nfSet - ofSet
	missingFiles = ofSet - nfSet
	for f in sorted(missingFiles):
		#matchText = colorama.Style.BRIGHT + colorama.Back.CYAN + "[ MISSING]" + colorama.Back.RESET + colorama.Style.RESET_ALL
		#print("{} Expected output file not generated: {}".format(matchText,f))
		indexHtml.append("
MISSING	{0}	diff results	output file	expected output file
".format(os.path.basename(f),os.path.basename(HTML_PATH),os.path.basename(OUTPUT_PATH),os.path.basename(EXPECTED_PATH)))

	# Index HTML Footer
	indexHtml.append("
")
	indexHtml.append("")
	indexHtml.append("")

	# Write out index.html
	parentDir = os.path.dirname(HTML_PATH)
	p = os.path.join(parentDir,"results.html")
	htmlout = open(p, mode='w', encoding="utf-8")
	htmlout.writelines(["%s\n" % item for item in indexHtml])
	htmlout.close()

	# Finished, summarize results
	print()
	if len(extraFiles) > 0:
		print("{} unexpected output files were generated.".format(len(extraFiles)))
	if len(missingFiles) > 0:
		print("{} expected output files were not generated.".format(len(missingFiles)))
	if fileChangeCount > 0:
		print("{} output file(s) differ with expected output, view diffout/results.html for diff results".format(fileChangeCount))
	else:
		print("No differences with expected output found.")
	print()

	return

def main():
	testDirectoryPath = "~/pp/tools/repo/diffout/"
	args = docopt(__doc__, version="diffout v{}".format(VERSION))

	#colorama.init()

	# Configure logging
	logLevel = logging.INFO #default
	if args['--verbose']:
		logLevel = logging.DEBUG
	elif args['--quiet']:
		logLevel = logging.ERROR

	logging.basicConfig(format='%(levelname)s: %(message)s', level=logLevel)
	logging.debug(args)

	# Make directories if missing
	if not os.path.exists(EXPECTED_PATH):
		os.makedirs(EXPECTED_PATH)
	if not os.path.exists(OUTPUT_PATH):
		os.makedirs(OUTPUT_PATH)
	if not os.path.exists(HTML_PATH):
		os.makedirs(HTML_PATH)
	if not os.path.exists(TERMINAL_OUT_PATH):
		os.makedirs(TERMINAL_OUT_PATH)

	# Save command
	if args['--save']:
		p = expandPath(EXPECTED_PATH)
		if os.path.exists(p):
			logging.info("--- Clearing current expected results")
			print(p)
			shutil.rmtree(p)
			os.makedirs(p)

		logging.info("--- Saving latest results to expected")
		saveFiles(getDirectoryFileList(OUTPUT_PATH),EXPECTED_PATH)
		return

	# Delete output files from previous run
	p = expandPath(OUTPUT_PATH)
	if os.path.exists(p):
		logging.info("--- Clearing results from last run")
		print(p)
		shutil.rmtree(p)
		os.makedirs(p)

	# Delete diff from previous run
	p = expandPath(HTML_PATH)
	if os.path.exists(p):
		logging.info("--- Clearing diffs from last run")
		print(p)
		shutil.rmtree(p)
		os.makedirs(p)

	# Delete terminal output files from previous run
	p = expandPath(TERMINAL_OUT_PATH)
	if os.path.exists(p):
		logging.info("--- Clearing terminal output logs from last run")
		print(p)
		shutil.rmtree(p)
		os.makedirs(p)

	# Write marker file for time index
	f = open("STARTTIME",'w')
	f.write('.')
	f.close()
	time.sleep(1)

	# Run command on each input file
	commandCount = 0
	commandErrorCount = 0
	for infile in args['<infile>']:
		for f in glob.glob(infile):
			commandline = args['<commandline>']
			commandline = commandline.replace('%F',f)
			terminalOutFile = open(os.path.join(TERMINAL_OUT_PATH,os.path.basename(f))+".out",'w')

			#s = colorama.Fore.LIGHTYELLOW_EX + "\n----- Running command:\n{}\n".format(commandline) + colorama.Fore.RESET
			s = "\n----- Running command:\n{}".format(commandline)
			print(s)
			commandCount += 1

			with terminalOutFile as outFile:
				cl = shlex.split(commandline)
				logging.debug("args: {}".format(str(cl)))
				if args['--pipe']:
					proc=subprocess.Popen(cl, stdout=outFile, stderr=outFile)
				else:
					proc=subprocess.Popen(cl)
				proc.wait()
				if( proc.returncode != 0 ):
					logging.error("Command failed: {}".format(commandline))
					commandErrorCount += 1

	# Copy recently modified files into output/
	outFiles = getFilesModifiedAfterFile("STARTTIME")
	saveFiles(outFiles,OUTPUT_PATH)

	print("\nFinished executing {} command(s) ({} error(s) occured).".format(commandCount,commandErrorCount))
	print("{} output file(s) were generated ({} expected).".format(len(getDirectoryFileList(OUTPUT_PATH)),len(getDirectoryFileList(EXPECTED_PATH))))
	print()

	diffDir(OUTPUT_PATH,EXPECTED_PATH)

	return

htmlHeader = ('<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">',
			  '',
			  '',
			  '',
			  ' <meta http-equiv="Content-Type" content="text/html; charset=UTF-8" />',
			  ' <title>diffout results</title>',
			  ' ',
			  '',
			  '',
			  '')

htmlFooter = ('
',
             '
  Legends
',
             '

',
             '
 Colors
',
             '
 Added
',
             '
Changed
',
             '
Deleted
',
             '
	',
             '
',
             '
 Links
',
             '
(f)irst change
',
             '
(n)ext change
',
             '
(t)op
',
             '
',
             '
',
             '
',
             '',
             '',
             '')

if __name__ == "__main__":
	main()
<|endoftext|>


<commit_before>#!/usr/bin/env python
#Copyright (c) 2011, 2012 Walter Bender
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA

from gettext import gettext as _

try:
    from numpy import append
    from numpy.fft import rfft
    PITCH_AVAILABLE = True
except:
    PITCH_AVAILABLE = False

from plugins.plugin import Plugin

from plugins.audio_sensors.audiograb import AudioGrab, \
    SENSOR_DC_NO_BIAS, SENSOR_DC_BIAS, SENSOR_AC_BIAS

from plugins.audio_sensors.ringbuffer import RingBuffer1d

from TurtleArt.tapalette import make_palette
from TurtleArt.taconstants import XO1, XO15, XO175, XO30
from TurtleArt.talogo import primitive_dictionary
from TurtleArt.tautils import debug_output

import logging
_logger = logging.getLogger('turtleart-activity audio sensors plugin')

def _avg(array, abs_value=False):
    ''' Calc. the average value of an array '''
    if len(array) == 0:
        return 0
    array_sum = 0
    if abs_value:
        for a in array:
            array_sum += abs(a)
    else:
        for a in array:
            array_sum += a
    return float(array_sum) / len(array)

class Audio_sensors(Plugin):

    def __init__(self, parent):
        self._parent = parent
        self._status = True  # TODO: test for audio device
        # These flags are referenced by audiograb
        self.hw = self._parent.hw
        self.running_sugar = self._parent.running_sugar

    def setup(self):
        ''' set up audio-sensor-specific blocks '''
        self.max_samples = 1500
        self.input_step = 1

        self.ringbuffer = []

        palette = make_palette('sensor',
                               colors=["#FF6060", "#A06060"],
                               help_string=_('Palette of sensor blocks'))

        primitive_dictionary['sound'] = self.prim_sound
        primitive_dictionary['volume'] = self.prim_volume
        if self._status:
            palette.add_block('sound',
                              style='box-style',
                              label=_('sound'),
                              help_string=_('raw microphone input signal'),
                              value_block=True,
                              prim_name='sound')

            palette.add_block('volume',
                              style='box-style',
                              label=_('loudness'),
                              help_string=_('microphone input volume'),
                              value_block=True,
                              prim_name='volume')
        else:
            palette.add_block('sound',
                              hidden=True,
                              style='box-style',
                              label=_('sound'),
                              help_string=_('raw microphone input signal'),
                              value_block=True,
                              prim_name='sound')
            palette.add_block('volume',
                              hidden=True,
                              style='box-style',
                              label=_('loudness'),
                              help_string=_('microphone input volume'),
                              value_block=True,
                              prim_name='volume')

        self._parent.lc.def_prim(
            'sound', 0, lambda self: primitive_dictionary['sound'](0))
        self._parent.lc.def_prim(
            'volume', 0, lambda self: primitive_dictionary['volume'](0))

        primitive_dictionary['pitch'] = self.prim_pitch
        if PITCH_AVAILABLE and self._status:
            palette.add_block('pitch',
                              style='box-style',
                              label=_('pitch'),
                              help_string=_('microphone input pitch'),
                              value_block=True,
                              prim_name='pitch')
        else:
            palette.add_block('pitch',
                              hidden=True,
                              style='box-style',
                              label=_('pitch'),
                              help_string=_('microphone input pitch'),
                              value_block=True,
                              prim_name='pitch')
        self._parent.lc.def_prim('pitch', 0,
                                 lambda self: primitive_dictionary['pitch'](0))

        primitive_dictionary['resistance'] = self.prim_resistance
        primitive_dictionary['voltage'] = self.prim_voltage
        if self.hw in [XO1, XO15, XO175, XO30] and self._status:
            if self.hw == XO1:
                self.voltage_gain = 0.00002225
                self.voltage_bias = 1.140
            elif self.hw == XO15:
                self.voltage_gain = -0.0001471
                self.voltage_bias = 1.695
            elif self.hw == XO175:
                self.voltage_gain = 0.000051
                self.voltage_bias = 1.372
            else:  # XO 3.0
                self.voltage_gain = 0.00007692
                self.voltage_bias = 0.719
            palette.add_block('resistance',
                              style='box-style',
                              label=_('resistance'),
                              help_string=_('microphone input resistance'),
                              value_block=True,
                              prim_name='resistance')
            palette.add_block('voltage',
                              style='box-style',
                              label=_('voltage'),
                              help_string=_('microphone input voltage'),
                              value_block=True,
                              prim_name='voltage')
        else:
            palette.add_block('resistance',
                              hidden=True,
                              style='box-style',
                              label=_('resistance'),
                              help_string=_('microphone input resistance'),
                              prim_name='resistance')
            palette.add_block('voltage',
                              hidden=True,
                              style='box-style',
                              label=_('voltage'),
                              help_string=_('microphone input voltage'),
                              prim_name='voltage')

        if self.hw in [XO15, XO175, XO30] and self._status:
            palette.add_block('resistance2',
                              style='box-style',
                              label=_('resistance') + '2',
                              help_string=_('microphone input resistance'),
                              value_block=True,
                              prim_name='resistance2')
            palette.add_block('voltage2',
                              style='box-style',
                              label=_('voltage') + '2',
                              help_string=_('microphone input voltage'),
                              value_block=True,
                              prim_name='voltage2')
        else:
            palette.add_block('resistance2',
                              hidden=True,
                              style='box-style',
                              label=_('resistance') + '2',
                              help_string=_('microphone input resistance'),
                              prim_name='resistance2')
            palette.add_block('voltage2',
                              hidden=True,
                              style='box-style',
                              label=_('voltage') + '2',
                              help_string=_('microphone input voltage'),
                              prim_name='voltage2')
        self._parent.lc.def_prim(
            'resistance', 0,
            lambda self: primitive_dictionary['resistance'](0))
        self._parent.lc.def_prim(
            'voltage', 0, lambda self: primitive_dictionary['voltage'](0))
        self._parent.lc.def_prim(
            'resistance2', 0,
            lambda self: primitive_dictionary['resistance'](1))
        self._parent.lc.def_prim(
            'voltage2', 0, lambda self: primitive_dictionary['voltage'](1))

        self.audio_started = False
        if self.hw in [XO175, XO30]:
            self.PARAMETERS = {
                SENSOR_AC_BIAS: (False, True, 80, True),
                SENSOR_DC_NO_BIAS: (True, False, 80, False),
                SENSOR_DC_BIAS: (True, True, 90, False)
                }
        elif self.hw == XO15:
            self.PARAMETERS = {
                SENSOR_AC_BIAS: (False, True, 80, True),
                SENSOR_DC_NO_BIAS: (True, False, 80, False),
                SENSOR_DC_BIAS: (True, True, 90, False)
                }
        elif self.hw == XO1:
            self.PARAMETERS = {
                SENSOR_AC_BIAS: (False, True, 40, True),
                SENSOR_DC_NO_BIAS: (True, False, 0, False),
                SENSOR_DC_BIAS: (True, True, 0, False)
                }
        else:
            self.PARAMETERS = {
                SENSOR_AC_BIAS: (None, True, 40, True),
                SENSOR_DC_NO_BIAS: (True, False, 80, False),
                SENSOR_DC_BIAS: (True, True, 90, False)
                }

    def start(self):
        ''' Start grabbing audio if there is an audio block in use '''
        if not self._status:
            return
        if self.audio_started:
            self.audiograb.stop_grabbing()
        if len(self._parent.block_list.get_similar_blocks(
                'block', ['volume', 'sound', 'pitch'])) > 0:
            mode, bias, gain, boost = self.PARAMETERS[SENSOR_AC_BIAS]
        elif len(self._parent.block_list.get_similar_blocks(
                'block', ['resistance', 'resistance2'])) > 0:
            mode, bias, gain, boost = self.PARAMETERS[SENSOR_DC_BIAS]
        elif len(self._parent.block_list.get_similar_blocks(
                'block', ['voltage', 'voltage2'])) > 0:
            mode, bias, gain, boost = self.PARAMETERS[SENSOR_DC_NO_BIAS]
        else:
            return  # No audio blocks in use.
        self.audiograb = AudioGrab(self.new_buffer, self,
                                   mode, bias, gain, boost)
        self._channels = self.audiograb.channels
        for i in range(self._channels):
            self.ringbuffer.append(RingBuffer1d(self.max_samples,
                                                dtype='int16'))
        self.audiograb.start_grabbing()
        self.audio_started = True

    def new_buffer(self, buf, channel=0):
        ''' Append a new buffer to the ringbuffer '''
        self.ringbuffer[channel].append(buf)
        return True

    def stop(self):
        ''' This gets called by the stop button '''
        if self._status and self.audio_started:
            self.audiograb.on_activity_quit()  # reset all setting
        self.audio_started = False

    def goto_background(self):
        ''' This gets called when your process is sent to the background '''
        pass

    def return_to_foreground(self):
        ''' This gets called when your process returns from the background '''
        pass

    def quit(self):
        ''' This gets called by the quit button '''
        if self._status and self.audio_started:
            self.audiograb.on_activity_quit()

    def _status_report(self):
        debug_output(
            'Reporting audio sensor status: %s' % (str(self._status)),
            self._parent.running_sugar)
        return self._status

    # Block primitives used in talogo

    def prim_volume(self, channel):
        ''' return mic in value '''
        if not self._status:
            return 0
        buf = self.ringbuffer[channel].read(None, self.input_step)
        if len(buf) > 0:
            volume = float(_avg(buf, abs_value=True))
            self._parent.lc.update_label_value('volume', volume)
            return volume
        else:
            return 0

    def prim_sound(self, channel):
        ''' return raw mic in value '''
        if not self._status:
            return 0
        buf = self.ringbuffer[channel].read(None, self.input_step)
        if len(buf) > 0:
            sound = float(buf[0])
            self._parent.lc.update_label_value('sound', sound)
            return sound
        else:
            return 0

    def prim_pitch(self, channel):
        ''' return index of max value in fft of mic in values '''
        if not PITCH_AVAILABLE or not self._status:
            return 0
        buf = []
        for i in range(4):
            buf = append(
                buf, self.ringbuffer[channel].read(None, self.input_step))
        if len(buf) > 0:
            r = []
            for j in rfft(buf):
                r.append(abs(j))
            # Convert output to Hertz
            pitch = r.index(max(r)) * 48000 / len(buf)
            self._parent.lc.update_label_value('pitch', pitch)
            return pitch
        else:
            return 0

    def prim_resistance(self, channel):
        ''' return resistance sensor value '''
        if not self.hw in [XO1, XO15, XO175, XO30] or not self._status:
            return 0
        if self.hw == XO1 and channel != 0:
            return 0
        buf = self.ringbuffer[channel].read(None, self.input_step)
        if len(buf) > 0:
            # See <http://bugs.sugarlabs.org/ticket/552#comment:7>
            # TODO: test this calibration on XO 1.5, XO 1.75
            avg_buf = float(_avg(buf))
            if self.hw == XO1:
                resistance = 2.718 ** ((avg_buf * 0.000045788) + 8.0531)
            elif self.hw == XO15:
                if avg_buf > 0:
                    resistance = (420000000 / avg_buf) - 13500
                else:
                    resistance = 420000000
            elif self.hw == XO175:
                if avg_buf < 30700:
                    return (180000000 / (30700 - avg_buf)) - 3150
                else:
                    resistance = 999999999
            else:  # XO 3.0
                if avg_buf < 30514:
                    return (46000000 / (30514 - avg_buf)) - 1150
                else:
                    resistance = 999999999
            if channel == 0:
                self._parent.lc.update_label_value('resistance', resistance)
            else:
                self._parent.lc.update_label_value('resistance2', resistance)
            if resistance < 0:
                resistance = 0
            return resistance
        else:
            return 0

    def prim_voltage(self, channel):
        ''' return voltage sensor value '''
        if not self.hw in [XO1, XO15, XO175, XO30] or not self._status:
            return 0
        if self.hw == XO1 and channel != 0:
            return 0
        buf = self.ringbuffer[channel].read(None, self.input_step)
        if len(buf) > 0:
            # See <http://bugs.sugarlabs.org/ticket/552#comment:7>
            voltage = float(_avg(buf)) * self.voltage_gain + self.voltage_bias
            if channel == 0:
                self._parent.lc.update_label_value('voltage', voltage)
            else:
                self._parent.lc.update_label_value('voltage2', voltage)
            return voltage
        else:
            return 0
<commit_msg>fix probelm where resistance labels not always updated<commit_after>#!/usr/bin/env python
#Copyright (c) 2011, 2012 Walter Bender
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA

from gettext import gettext as _

try:
    from numpy import append
    from numpy.fft import rfft
    PITCH_AVAILABLE = True
except:
    PITCH_AVAILABLE = False

from plugins.plugin import Plugin

from plugins.audio_sensors.audiograb import AudioGrab, \
    SENSOR_DC_NO_BIAS, SENSOR_DC_BIAS, SENSOR_AC_BIAS

from plugins.audio_sensors.ringbuffer import RingBuffer1d

from TurtleArt.tapalette import make_palette
from TurtleArt.taconstants import XO1, XO15, XO175, XO30
from TurtleArt.talogo import primitive_dictionary
from TurtleArt.tautils import debug_output

import logging
_logger = logging.getLogger('turtleart-activity audio sensors plugin')

def _avg(array, abs_value=False):
    ''' Calc. the average value of an array '''
    if len(array) == 0:
        return 0
    array_sum = 0
    if abs_value:
        for a in array:
            array_sum += abs(a)
    else:
        for a in array:
            array_sum += a
    return float(array_sum) / len(array)

class Audio_sensors(Plugin):

    def __init__(self, parent):
        self._parent = parent
        self._status = True  # TODO: test for audio device
        # These flags are referenced by audiograb
        self.hw = self._parent.hw
        self.running_sugar = self._parent.running_sugar

    def setup(self):
        ''' set up audio-sensor-specific blocks '''
        self.max_samples = 1500
        self.input_step = 1

        self.ringbuffer = []

        palette = make_palette('sensor',
                               colors=["#FF6060", "#A06060"],
                               help_string=_('Palette of sensor blocks'))

        primitive_dictionary['sound'] = self.prim_sound
        primitive_dictionary['volume'] = self.prim_volume
        if self._status:
            palette.add_block('sound',
                              style='box-style',
                              label=_('sound'),
                              help_string=_('raw microphone input signal'),
                              value_block=True,
                              prim_name='sound')

            palette.add_block('volume',
                              style='box-style',
                              label=_('loudness'),
                              help_string=_('microphone input volume'),
                              value_block=True,
                              prim_name='volume')
        else:
            palette.add_block('sound',
                              hidden=True,
                              style='box-style',
                              label=_('sound'),
                              help_string=_('raw microphone input signal'),
                              value_block=True,
                              prim_name='sound')
            palette.add_block('volume',
                              hidden=True,
                              style='box-style',
                              label=_('loudness'),
                              help_string=_('microphone input volume'),
                              value_block=True,
                              prim_name='volume')

        self._parent.lc.def_prim(
            'sound', 0, lambda self: primitive_dictionary['sound'](0))
        self._parent.lc.def_prim(
            'volume', 0, lambda self: primitive_dictionary['volume'](0))

        primitive_dictionary['pitch'] = self.prim_pitch
        if PITCH_AVAILABLE and self._status:
            palette.add_block('pitch',
                              style='box-style',
                              label=_('pitch'),
                              help_string=_('microphone input pitch'),
                              value_block=True,
                              prim_name='pitch')
        else:
            palette.add_block('pitch',
                              hidden=True,
                              style='box-style',
                              label=_('pitch'),
                              help_string=_('microphone input pitch'),
                              value_block=True,
                              prim_name='pitch')
        self._parent.lc.def_prim('pitch', 0,
                                 lambda self: primitive_dictionary['pitch'](0))

        primitive_dictionary['resistance'] = self.prim_resistance
        primitive_dictionary['voltage'] = self.prim_voltage
        if self.hw in [XO1, XO15, XO175, XO30] and self._status:
            if self.hw == XO1:
                self.voltage_gain = 0.00002225
                self.voltage_bias = 1.140
            elif self.hw == XO15:
                self.voltage_gain = -0.0001471
                self.voltage_bias = 1.695
            elif self.hw == XO175:
                self.voltage_gain = 0.000051
                self.voltage_bias = 1.372
            else:  # XO 3.0
                self.voltage_gain = 0.00007692
                self.voltage_bias = 0.719
            palette.add_block('resistance',
                              style='box-style',
                              label=_('resistance'),
                              help_string=_('microphone input resistance'),
                              value_block=True,
                              prim_name='resistance')
            palette.add_block('voltage',
                              style='box-style',
                              label=_('voltage'),
                              help_string=_('microphone input voltage'),
                              value_block=True,
                              prim_name='voltage')
        else:
            palette.add_block('resistance',
                              hidden=True,
                              style='box-style',
                              label=_('resistance'),
                              help_string=_('microphone input resistance'),
                              prim_name='resistance')
            palette.add_block('voltage',
                              hidden=True,
                              style='box-style',
                              label=_('voltage'),
                              help_string=_('microphone input voltage'),
                              prim_name='voltage')

        if self.hw in [XO15, XO175, XO30] and self._status:
            palette.add_block('resistance2',
                              style='box-style',
                              label=_('resistance') + '2',
                              help_string=_('microphone input resistance'),
                              value_block=True,
                              prim_name='resistance2')
            palette.add_block('voltage2',
                              style='box-style',
                              label=_('voltage') + '2',
                              help_string=_('microphone input voltage'),
                              value_block=True,
                              prim_name='voltage2')
        else:
            palette.add_block('resistance2',
                              hidden=True,
                              style='box-style',
                              label=_('resistance') + '2',
                              help_string=_('microphone input resistance'),
                              prim_name='resistance2')
            palette.add_block('voltage2',
                              hidden=True,
                              style='box-style',
                              label=_('voltage') + '2',
                              help_string=_('microphone input voltage'),
                              prim_name='voltage2')
        self._parent.lc.def_prim(
            'resistance', 0,
            lambda self: primitive_dictionary['resistance'](0))
        self._parent.lc.def_prim(
            'voltage', 0, lambda self: primitive_dictionary['voltage'](0))
        self._parent.lc.def_prim(
            'resistance2', 0,
            lambda self: primitive_dictionary['resistance'](1))
        self._parent.lc.def_prim(
            'voltage2', 0, lambda self: primitive_dictionary['voltage'](1))

        self.audio_started = False
        if self.hw in [XO175, XO30]:
            self.PARAMETERS = {
                SENSOR_AC_BIAS: (False, True, 80, True),
                SENSOR_DC_NO_BIAS: (True, False, 80, False),
                SENSOR_DC_BIAS: (True, True, 90, False)
                }
        elif self.hw == XO15:
            self.PARAMETERS = {
                SENSOR_AC_BIAS: (False, True, 80, True),
                SENSOR_DC_NO_BIAS: (True, False, 80, False),
                SENSOR_DC_BIAS: (True, True, 90, False)
                }
        elif self.hw == XO1:
            self.PARAMETERS = {
                SENSOR_AC_BIAS: (False, True, 40, True),
                SENSOR_DC_NO_BIAS: (True, False, 0, False),
                SENSOR_DC_BIAS: (True, True, 0, False)
                }
        else:
            self.PARAMETERS = {
                SENSOR_AC_BIAS: (None, True, 40, True),
                SENSOR_DC_NO_BIAS: (True, False, 80, False),
                SENSOR_DC_BIAS: (True, True, 90, False)
                }

    def start(self):
        ''' Start grabbing audio if there is an audio block in use '''
        if not self._status:
            return
        if self.audio_started:
            self.audiograb.stop_grabbing()
        if len(self._parent.block_list.get_similar_blocks(
                'block', ['volume', 'sound', 'pitch'])) > 0:
            mode, bias, gain, boost = self.PARAMETERS[SENSOR_AC_BIAS]
        elif len(self._parent.block_list.get_similar_blocks(
                'block', ['resistance', 'resistance2'])) > 0:
            mode, bias, gain, boost = self.PARAMETERS[SENSOR_DC_BIAS]
        elif len(self._parent.block_list.get_similar_blocks(
                'block', ['voltage', 'voltage2'])) > 0:
            mode, bias, gain, boost = self.PARAMETERS[SENSOR_DC_NO_BIAS]
        else:
            return  # No audio blocks in use.
        self.audiograb = AudioGrab(self.new_buffer, self,
                                   mode, bias, gain, boost)
        self._channels = self.audiograb.channels
        for i in range(self._channels):
            self.ringbuffer.append(RingBuffer1d(self.max_samples,
                                                dtype='int16'))
        self.audiograb.start_grabbing()
        self.audio_started = True

    def new_buffer(self, buf, channel=0):
        ''' Append a new buffer to the ringbuffer '''
        self.ringbuffer[channel].append(buf)
        return True

    def stop(self):
        ''' This gets called by the stop button '''
        if self._status and self.audio_started:
            self.audiograb.on_activity_quit()  # reset all setting
        self.audio_started = False

    def goto_background(self):
        ''' This gets called when your process is sent to the background '''
        pass

    def return_to_foreground(self):
        ''' This gets called when your process returns from the background '''
        pass

    def quit(self):
        ''' This gets called by the quit button '''
        if self._status and self.audio_started:
            self.audiograb.on_activity_quit()

    def _status_report(self):
        debug_output(
            'Reporting audio sensor status: %s' % (str(self._status)),
            self._parent.running_sugar)
        return self._status

    # Block primitives used in talogo

    def prim_volume(self, channel):
        ''' return mic in value '''
        if not self._status:
            return 0
        buf = self.ringbuffer[channel].read(None, self.input_step)
        if len(buf) > 0:
            volume = float(_avg(buf, abs_value=True))
            self._parent.lc.update_label_value('volume', volume)
            return volume
        else:
            return 0

    def prim_sound(self, channel):
        ''' return raw mic in value '''
        if not self._status:
            return 0
        buf = self.ringbuffer[channel].read(None, self.input_step)
        if len(buf) > 0:
            sound = float(buf[0])
            self._parent.lc.update_label_value('sound', sound)
            return sound
        else:
            return 0

    def prim_pitch(self, channel):
        ''' return index of max value in fft of mic in values '''
        if not PITCH_AVAILABLE or not self._status:
            return 0
        buf = []
        for i in range(4):
            buf = append(
                buf, self.ringbuffer[channel].read(None, self.input_step))
        if len(buf) > 0:
            r = []
            for j in rfft(buf):
                r.append(abs(j))
            # Convert output to Hertz
            pitch = r.index(max(r)) * 48000 / len(buf)
            self._parent.lc.update_label_value('pitch', pitch)
            return pitch
        else:
            return 0

    def prim_resistance(self, channel):
        ''' return resistance sensor value '''
        if not self.hw in [XO1, XO15, XO175, XO30] or not self._status:
            return 0
        if self.hw == XO1 and channel != 0:
            return 0
        buf = self.ringbuffer[channel].read(None, self.input_step)
        if len(buf) > 0:
            # See <http://bugs.sugarlabs.org/ticket/552#comment:7>
            # TODO: test this calibration on XO 1.5, XO 1.75
            avg_buf = float(_avg(buf))
            if self.hw == XO1:
                resistance = 2.718 ** ((avg_buf * 0.000045788) + 8.0531)
            elif self.hw == XO15:
                if avg_buf > 0:
                    resistance = (420000000 / avg_buf) - 13500
                else:
                    resistance = 420000000
            elif self.hw == XO175:
                if avg_buf < 30700:
                    resistance = (180000000 / (30700 - avg_buf)) - 3150
                else:
                    resistance = 999999999
            else:  # XO 3.0
                if avg_buf < 30514:
                    resistance = (46000000 / (30514 - avg_buf)) - 1150
                else:
                    resistance = 999999999
            if channel == 0:
                self._parent.lc.update_label_value('resistance', resistance)
            else:
                self._parent.lc.update_label_value('resistance2', resistance)
            if resistance < 0:
                resistance = 0
            return resistance
        else:
            return 0

    def prim_voltage(self, channel):
        ''' return voltage sensor value '''
        if not self.hw in [XO1, XO15, XO175, XO30] or not self._status:
            return 0
        if self.hw == XO1 and channel != 0:
            return 0
        buf = self.ringbuffer[channel].read(None, self.input_step)
        if len(buf) > 0:
            # See <http://bugs.sugarlabs.org/ticket/552#comment:7>
            voltage = float(_avg(buf)) * self.voltage_gain + self.voltage_bias
            if channel == 0:
                self._parent.lc.update_label_value('voltage', voltage)
            else:
                self._parent.lc.update_label_value('voltage2', voltage)
            return voltage
        else:
            return 0
<|endoftext|>


<commit_before>from __future__ import print_function
from __future__ import absolute_import
from __future__ import unicode_literals
import re
from collections import defaultdict
from github import Github, UnknownObjectException
from django.template.loader import render_to_string
import requests
from gevent.pool import Pool
from corehq.util.soft_assert import soft_assert

LABELS_TO_EXPAND = [
    "product/all-users-all-environments",
    "product/prod-india-all-users",
    "product/feature-flag",
    "product/all-users",
]

def get_deploy_email_message_body(user, compare_url):
    try:
        ref_comparison = compare_url.split('/')[-1]
        last_deploy, current_deploy = ref_comparison.split('...')
    except ValueError:
        # Not a real compare_url
        last_deploy = current_deploy = 'Unavailable'
        pr_infos = []
    else:
        pr_numbers = _get_pr_numbers(last_deploy, current_deploy)
        pool = Pool(5)
        pr_infos = [_f for _f in pool.map(_get_pr_info, pr_numbers) if _f]

    prs_by_label = _get_prs_by_label(pr_infos)

    return render_to_string('hqadmin/partials/project_snapshot.html', {
        'pr_merges': pr_infos,
        'prs_by_label': prs_by_label,
        'last_deploy': last_deploy,
        'current_deploy': current_deploy,
        'user': user,
        'compare_url': compare_url,
    })

def _get_pr_numbers(last_deploy, current_deploy):
    gh = Github()
    repo = gh.get_organization('dimagi').get_repo('commcare-hq')
    try:
        last_deploy_sha = repo.get_commit(last_deploy).sha
        current_deploy_sha = repo.get_commit(current_deploy).sha
        comparison = repo.compare(last_deploy_sha, current_deploy_sha)
    except UnknownObjectException as e:
        print("An exception occurred while getting commits from the repo. This could be because of rate limiting")
        print("Please see: https://manage.dimagi.com/default.asp?266529 for more info.")
        print(e)
        rate = gh.get_rate_limit().rate
        print("\tWe have {} of {} requests left".format(rate.remaining, rate.limit))

        if rate.remaining == 0:
            _assert = soft_assert(to="@".join(['frener', 'dimagi.com']))
            message = "Rate limit reached when getting commits. See https://manage.dimagi.com/default.asp?266529"
            _assert(False, message, e)

        raise

    return [
        int(re.search(r'Merge pull request #(\d+)', repo_commit.commit.message).group(1))
        for repo_commit in comparison.commits
        if repo_commit.commit.message.startswith('Merge pull request')
    ]

def _get_pr_info(pr_number):
    url = 'https://api.github.com/repos/dimagi/commcare-hq/pulls/{}'.format(pr_number)
    json_response = requests.get(url).json()
    if not json_response.get('number'):
        # We've probably exceed our rate limit for unauthenticated requests
        return None
    assert pr_number == json_response['number'], (pr_number, json_response['number'])
    additions = json_response['additions']
    deletions = json_response['deletions']

    line_changes = additions + deletions

    labels = _get_pr_labels(pr_number)

    return {
        'number': json_response['number'],
        'title': json_response['title'],
        'body': json_response['body'],
        'opened_by': json_response['user']['login'],
        'url': json_response['html_url'],
        'merge_base': json_response['base']['label'],
        'merge_head': json_response['head']['label'],
        'additions': additions,
        'deletions': deletions,
        'line_changes': line_changes,
        'labels': labels,
    }

def _get_pr_labels(pr_number):
    url = 'https://api.github.com/repos/dimagi/commcare-hq/issues/{}'.format(pr_number)
    json_response = requests.get(url).json()
    if not json_response.get('number'):
        # We've probably exceed our rate limit for unauthenticated requests
        return []
    assert pr_number == json_response['number'], (pr_number, json_response['number'])

    return [
        {'name': label['name'], 'color': label['color']}
        for label in json_response['labels']
    ]

def _get_prs_by_label(pr_infos):
    prs_by_label = defaultdict(list)
    for pr in pr_infos:
        for label in pr['labels']:
            if label['name'] in LABELS_TO_EXPAND:
                prs_by_label[label['name']].append(pr)
    return dict(prs_by_label)

if __name__ == '__main__':
    def setup_fake_django():
        from django.conf import settings
        import os
        settings.configure(
            TEMPLATE_LOADERS=('django.template.loaders.filesystem.Loader',),
            TEMPLATE_DIRS=(os.path.join(os.path.dirname(__file__), '..', 'templates'),),
        )
    import sys
    if sys.argv[1] == 'get_deploy_email_message_body':
        setup_fake_django()
        print(get_deploy_email_message_body(*sys.argv[3:5]).encode('utf-8'))
<commit_msg>Revert "Add debug info to pr emails"<commit_after>from __future__ import print_function
from __future__ import absolute_import
from __future__ import unicode_literals
import re
from collections import defaultdict
from github import Github
from django.template.loader import render_to_string
import requests
from gevent.pool import Pool

LABELS_TO_EXPAND = [
    "product/all-users-all-environments",
    "product/prod-india-all-users",
    "product/feature-flag",
    "product/all-users",
]

def get_deploy_email_message_body(user, compare_url):
    try:
        ref_comparison = compare_url.split('/')[-1]
        last_deploy, current_deploy = ref_comparison.split('...')
    except ValueError:
        # Not a real compare_url
        last_deploy = current_deploy = 'Unavailable'
        pr_infos = []
    else:
        pr_numbers = _get_pr_numbers(last_deploy, current_deploy)
        pool = Pool(5)
        pr_infos = [_f for _f in pool.map(_get_pr_info, pr_numbers) if _f]

    prs_by_label = _get_prs_by_label(pr_infos)

    return render_to_string('hqadmin/partials/project_snapshot.html', {
        'pr_merges': pr_infos,
        'prs_by_label': prs_by_label,
        'last_deploy': last_deploy,
        'current_deploy': current_deploy,
        'user': user,
        'compare_url': compare_url,
    })

def _get_pr_numbers(last_deploy, current_deploy):
    repo = Github().get_organization('dimagi').get_repo('commcare-hq')
    last_deploy_sha = repo.get_commit(last_deploy).sha
    current_deploy_sha = repo.get_commit(current_deploy).sha
    comparison = repo.compare(last_deploy_sha, current_deploy_sha)

    return [
        int(re.search(r'Merge pull request #(\d+)', repo_commit.commit.message).group(1))
        for repo_commit in comparison.commits
        if repo_commit.commit.message.startswith('Merge pull request')
    ]

def _get_pr_info(pr_number):
    url = 'https://api.github.com/repos/dimagi/commcare-hq/pulls/{}'.format(pr_number)
    json_response = requests.get(url).json()
    if not json_response.get('number'):
        # We've probably exceed our rate limit for unauthenticated requests
        return None
    assert pr_number == json_response['number'], (pr_number, json_response['number'])
    additions = json_response['additions']
    deletions = json_response['deletions']

    line_changes = additions + deletions

    labels = _get_pr_labels(pr_number)

    return {
        'number': json_response['number'],
        'title': json_response['title'],
        'body': json_response['body'],
        'opened_by': json_response['user']['login'],
        'url': json_response['html_url'],
        'merge_base': json_response['base']['label'],
        'merge_head': json_response['head']['label'],
        'additions': additions,
        'deletions': deletions,
        'line_changes': line_changes,
        'labels': labels,
    }

def _get_pr_labels(pr_number):
    url = 'https://api.github.com/repos/dimagi/commcare-hq/issues/{}'.format(pr_number)
    json_response = requests.get(url).json()
    if not json_response.get('number'):
        # We've probably exceed our rate limit for unauthenticated requests
        return []
    assert pr_number == json_response['number'], (pr_number, json_response['number'])

    return [
        {'name': label['name'], 'color': label['color']}
        for label in json_response['labels']
    ]

def _get_prs_by_label(pr_infos):
    prs_by_label = defaultdict(list)
    for pr in pr_infos:
        for label in pr['labels']:
            if label['name'] in LABELS_TO_EXPAND:
                prs_by_label[label['name']].append(pr)
    return dict(prs_by_label)

if __name__ == '__main__':
    def setup_fake_django():
        from django.conf import settings
        import os
        settings.configure(
            TEMPLATE_LOADERS=('django.template.loaders.filesystem.Loader',),
            TEMPLATE_DIRS=(os.path.join(os.path.dirname(__file__), '..', 'templates'),),
        )
    import sys
    if sys.argv[1] == 'get_deploy_email_message_body':
        setup_fake_django()
        print(get_deploy_email_message_body(*sys.argv[3:5]).encode('utf-8'))
<|endoftext|>


<commit_before>import os.path as op
import numpy as np
import nibabel as nib
import dipy.data as dpd
import dipy.tracking.utils as dtu
import dipy.tracking.streamline as dts
from dipy.io.streamline import save_tractogram, load_tractogram
from dipy.stats.analysis import afq_profile, gaussian_weights
from dipy.io.stateful_tractogram import StatefulTractogram
from dipy.io.stateful_tractogram import Space
import dipy.core.gradients as dpg
from dipy.segment.mask import median_otsu

import AFQ.data as afd
import AFQ.tractography as aft
import AFQ.registration as reg
import AFQ.dti as dti
import AFQ.segmentation as seg

afd.fetch_hcp([subject])
afd.afq_home

dwi_dir = op.join(afd.afq_home, 'HCP', 'derivatives',
                'dmriprep', f'sub-{subject}', 'sess-01/dwi')

hardi_fdata = op.join(dwi_dir, f"sub-{subject}_dwi.nii.gz")
hardi_fbval = op.join(dwi_dir, f"sub-{subject}_dwi.bval")
hardi_fbvec = op.join(dwi_dir, f"sub-{subject}_dwi.bvec")

img = nib.load(hardi_fdata)
gtab = dpg.gradient_table(hardi_fbval, hardi_fbvec)

print("Calculating brain-mask")
if not op.exists('./brain_mask.nii.gz'):
    mean_b0 = np.mean(img.get_data()[..., gtab.b0s_mask], -1)
    _, brain_mask = median_otsu(mean_b0, median_radius=4,
                                numpass=1, autocrop=False,
                                vol_idx=None, dilate=10)
    be_img = nib.Nifti1Image(brain_mask.astype(int),
                            img.affine)
    nib.save(be_img, './brain_mask.nii.gz')

print("Calculating DTI...")
if not op.exists('./dti_FA.nii.gz'):
    dti_params = dti.fit_dti(hardi_fdata, hardi_fbval, hardi_fbvec,
                            out_dir='.', b0_threshold=50,
                            mask='./brain_mask.nii.gz')
else:
    dti_params = {'FA': './dti_FA.nii.gz',
                'params': './dti_params.nii.gz'}

FA_img = nib.load(dti_params['FA'])
FA_data = FA_img.get_fdata()

print("Registering to template...")
MNI_T2_img = dpd.read_mni_template()
if not op.exists('mapping.nii.gz'):
    import dipy.core.gradients as dpg
    gtab = dpg.gradient_table(hardi_fbval, hardi_fbvec)
    warped_hardi, mapping = reg.syn_register_dwi(hardi_fdata, gtab,
                                                template=MNI_T2_img)
    reg.write_mapping(mapping, './mapping.nii.gz')
else:
    mapping = reg.read_mapping('./mapping.nii.gz', img, MNI_T2_img)

from AFQ import api

bundle_names = ['CST',
                'C',
                'F',
                'UF',
                'MCP',
                'AF',
                'CCMid',
                'AF',
                'CC_ForcepsMajor',
                'CC_ForcepsMinor',
                'IFOF']
bundles = api.make_bundle_dict(bundle_names=bundle_names, seg_algo="reco")

from AFQ import csd

print("Calculating CSD...")
if not op.exists('./csd_sh_coeff.nii.gz'):
    csd_params = csd.fit_csd(hardi_fdata, hardi_fbval, hardi_fbvec,
                            out_dir='.', b0_threshold=50,
                            mask='./brain_mask.nii.gz')
else:
    csd_params = './csd_sh_coeff.nii.gz'

print("Tracking...")
if not op.exists('csd_streamlines_reco.trk'):
    seed_roi = np.zeros(img.shape[:-1])
    seed_roi[FA_data > 0.4] = 1

    nib.save(nib.Nifti1Image(seed_roi, img.affine), 'seed_roi.nii.gz')
    streamlines = aft.track(csd_params, seed_mask=seed_roi,
                            directions='det', stop_mask=FA_data,
                            stop_threshold=0.1)
    print(len(streamlines))
    sft = StatefulTractogram(streamlines, img, Space.RASMM)
    save_tractogram(sft, './csd_streamlines_reco.trk',
                    bbox_valid_check=False)
else:
    tg = load_tractogram('./csd_streamlines_reco.trk', img)
    streamlines = tg.streamlines

print("Segmenting fiber groups...")
segmentation = seg.Segmentation(algo='reco',
                                model_clust_thr=20,
                                reduction_thr=20)
segmentation.segment(bundles, streamlines)
fiber_groups = segmentation.fiber_groups

for kk in fiber_groups:
    print(kk, len(fiber_groups[kk]))
    sft = StatefulTractogram(fiber_groups[kk], img, Space.RASMM)
    save_tractogram(sft, './%s_reco.trk'%kk,
                    bbox_valid_check=False)

<commit_msg>Warn that this takes a long time to run.<commit_after>import warnings
import os.path as op
import numpy as np
import nibabel as nib
import dipy.data as dpd
import dipy.tracking.utils as dtu
import dipy.tracking.streamline as dts
from dipy.io.streamline import save_tractogram, load_tractogram
from dipy.stats.analysis import afq_profile, gaussian_weights
from dipy.io.stateful_tractogram import StatefulTractogram
from dipy.io.stateful_tractogram import Space
import dipy.core.gradients as dpg
from dipy.segment.mask import median_otsu

import AFQ.data as afd
import AFQ.tractography as aft
import AFQ.registration as reg
import AFQ.dti as dti
import AFQ.segmentation as seg

warnings.warn(
    "This example requires a reasonably large amount of memory and takes approximately 3-4 hours to run")

afd.fetch_hcp([subject])
afd.afq_home

dwi_dir = op.join(afd.afq_home, 'HCP', 'derivatives',
                'dmriprep', f'sub-{subject}', 'sess-01/dwi')

hardi_fdata = op.join(dwi_dir, f"sub-{subject}_dwi.nii.gz")
hardi_fbval = op.join(dwi_dir, f"sub-{subject}_dwi.bval")
hardi_fbvec = op.join(dwi_dir, f"sub-{subject}_dwi.bvec")

img = nib.load(hardi_fdata)
gtab = dpg.gradient_table(hardi_fbval, hardi_fbvec)

print("Calculating brain-mask")
if not op.exists('./brain_mask.nii.gz'):
    mean_b0 = np.mean(img.get_data()[..., gtab.b0s_mask], -1)
    _, brain_mask = median_otsu(mean_b0, median_radius=4,
                                numpass=1, autocrop=False,
                                vol_idx=None, dilate=10)
    be_img = nib.Nifti1Image(brain_mask.astype(int),
                            img.affine)
    nib.save(be_img, './brain_mask.nii.gz')

print("Calculating DTI...")
if not op.exists('./dti_FA.nii.gz'):
    dti_params = dti.fit_dti(hardi_fdata, hardi_fbval, hardi_fbvec,
                            out_dir='.', b0_threshold=50,
                            mask='./brain_mask.nii.gz')
else:
    dti_params = {'FA': './dti_FA.nii.gz',
                'params': './dti_params.nii.gz'}

FA_img = nib.load(dti_params['FA'])
FA_data = FA_img.get_fdata()

print("Registering to template...")
MNI_T2_img = dpd.read_mni_template()
if not op.exists('mapping.nii.gz'):
    import dipy.core.gradients as dpg
    gtab = dpg.gradient_table(hardi_fbval, hardi_fbvec)
    warped_hardi, mapping = reg.syn_register_dwi(hardi_fdata, gtab,
                                                template=MNI_T2_img)
    reg.write_mapping(mapping, './mapping.nii.gz')
else:
    mapping = reg.read_mapping('./mapping.nii.gz', img, MNI_T2_img)

from AFQ import api

bundle_names = ['CST',
                'C',
                'F',
                'UF',
                'MCP',
                'AF',
                'CCMid',
                'AF',
                'CC_ForcepsMajor',
                'CC_ForcepsMinor',
                'IFOF']
bundles = api.make_bundle_dict(bundle_names=bundle_names, seg_algo="reco")

from AFQ import csd

print("Calculating CSD...")
if not op.exists('./csd_sh_coeff.nii.gz'):
    csd_params = csd.fit_csd(hardi_fdata, hardi_fbval, hardi_fbvec,
                            out_dir='.', b0_threshold=50,
                            mask='./brain_mask.nii.gz')
else:
    csd_params = './csd_sh_coeff.nii.gz'

print("Tracking...")
if not op.exists('csd_streamlines_reco.trk'):
    seed_roi = np.zeros(img.shape[:-1])
    seed_roi[FA_data > 0.4] = 1

    nib.save(nib.Nifti1Image(seed_roi, img.affine), 'seed_roi.nii.gz')
    streamlines = aft.track(csd_params, seed_mask=seed_roi,
                            directions='det', stop_mask=FA_data,
                            stop_threshold=0.1)
    print(len(streamlines))
    sft = StatefulTractogram(streamlines, img, Space.RASMM)
    save_tractogram(sft, './csd_streamlines_reco.trk',
                    bbox_valid_check=False)
else:
    tg = load_tractogram('./csd_streamlines_reco.trk', img)
    streamlines = tg.streamlines

print("Segmenting fiber groups...")
segmentation = seg.Segmentation(algo='reco',
                                model_clust_thr=20,
                                reduction_thr=20)
segmentation.segment(bundles, streamlines)
fiber_groups = segmentation.fiber_groups

for kk in fiber_groups:
    print(kk, len(fiber_groups[kk]))
    sft = StatefulTractogram(fiber_groups[kk], img, Space.RASMM)
    save_tractogram(sft, './%s_reco.trk'%kk,
                    bbox_valid_check=False)

<|endoftext|>