<commit_before>"""
Test that Gabble properly cleans up delayed RequestStream contexts
and returns an error when Disconnect is called and there are
incomplete requests.
"""

from gabbletest import exec_test, disconnect_conn
from servicetest import make_channel_proxy, call_async, sync_dbus, EventPattern
import jingletest

import constants as cs

def test(q, bus, conn, stream):
    jt = jingletest.JingleTest(stream, 'test@localhost', 'foo@bar.com/Foo')

    conn.Connect()

    q.expect('dbus-signal', signal='StatusChanged',
            args=[cs.CONN_STATUS_CONNECTING, cs.CSR_REQUESTED])

    q.expect('stream-authenticated')
    q.expect('dbus-signal', signal='PresenceUpdate',
        args=[{1L: (0L, {u'available': {}})}])
    q.expect('dbus-signal', signal='StatusChanged',
            args=[cs.CONN_STATUS_CONNECTED, cs.CSR_REQUESTED])

    # We intentionally DON'T set remote presence yet. Since Gabble is still
    # unsure whether to treat contact as offline for this purpose, it
    # will tentatively allow channel creation and contact handle addition

    handle = conn.RequestHandles(cs.HT_CONTACT, [jt.remote_jid])[0]

    path = conn.RequestChannel(cs.CHANNEL_TYPE_STREAMED_MEDIA, cs.HT_CONTACT,
        handle, True)
    media_iface = make_channel_proxy(conn, path, 'Channel.Type.StreamedMedia')

    # So it turns out that the calls to RequestStreams and Disconnect could be
    # reordered while the first waits on the result of introspecting the
    # channel's object which is kicked off by making a proxy object for it,
    # whereas the connection proxy is long ago introspected. Isn't dbus-python
    # great? Syncing here forces that introspection to finish so we can rely on
    # the ordering of RequestStreams and Disconnect. Yay.
    sync_dbus(bus, q, conn)

    # Now we request streams before either <presence> or caps have arrived
    call_async(q, media_iface, 'RequestStreams', handle,
        [cs.MEDIA_STREAM_TYPE_AUDIO])

    before_events, after_events = disconnect_conn(q, conn, stream,
        [EventPattern('dbus-error', method='RequestStreams')])

    # RequestStreams should now return NotAvailable
    assert before_events[0].error.get_dbus_name() == cs.NOT_AVAILABLE, \
            before_events[0].error

if __name__ == '__main__':
    exec_test(test, timeout=10)

<commit_msg>Update test-wait-for-caps-incomplete to handle Calls as well.<commit_after>"""
Test that Gabble properly cleans up delayed RequestStream contexts
and returns an error when Disconnect is called and there are
incomplete requests.
"""

from gabbletest import exec_test, disconnect_conn
from servicetest import make_channel_proxy, call_async, sync_dbus, EventPattern
import jingletest

import constants as cs

def test(q, bus, conn, stream, channel_type):
    jt = jingletest.JingleTest(stream, 'test@localhost', 'foo@bar.com/Foo')

    conn.Connect()

    q.expect('dbus-signal', signal='StatusChanged',
            args=[cs.CONN_STATUS_CONNECTING, cs.CSR_REQUESTED])

    q.expect('stream-authenticated')
    q.expect('dbus-signal', signal='PresenceUpdate',
        args=[{1L: (0L, {u'available': {}})}])
    q.expect('dbus-signal', signal='StatusChanged',
            args=[cs.CONN_STATUS_CONNECTED, cs.CSR_REQUESTED])

    # We intentionally DON'T set remote presence yet. Since Gabble is still
    # unsure whether to treat contact as offline for this purpose, it
    # will tentatively allow channel creation and contact handle addition

    handle = conn.RequestHandles(cs.HT_CONTACT, [jt.remote_jid])[0]

    if channel_type == cs.CHANNEL_TYPE_STREAMED_MEDIA:
        path = conn.RequestChannel(cs.CHANNEL_TYPE_STREAMED_MEDIA,
            cs.HT_CONTACT, handle, True)
        media_iface = make_channel_proxy(conn, path,
            'Channel.Type.StreamedMedia')

    # So it turns out that the calls to RequestStreams and Disconnect could be
    # reordered while the first waits on the result of introspecting the
    # channel's object which is kicked off by making a proxy object for it,
    # whereas the connection proxy is long ago introspected. Isn't dbus-python
    # great? Syncing here forces that introspection to finish so we can rely on
    # the ordering of RequestStreams and Disconnect. Yay.
    sync_dbus(bus, q, conn)

    # Now we request streams before either <presence> or caps have arrived
    if channel_type == cs.CHANNEL_TYPE_STREAMED_MEDIA:
        call_async(q, media_iface, 'RequestStreams', handle,
            [cs.MEDIA_STREAM_TYPE_AUDIO])

        before_events, after_events = disconnect_conn(q, conn, stream,
            [EventPattern('dbus-error', method='RequestStreams')])

        # RequestStreams should now return NotAvailable
        assert before_events[0].error.get_dbus_name() == cs.NOT_AVAILABLE, \
            before_events[0].error
    else:
        call_async(q, conn.Requests, 'CreateChannel',
            { cs.CHANNEL_TYPE: channel_type,
              cs.TARGET_HANDLE_TYPE: cs.HT_CONTACT,
              cs.TARGET_ID: jt.remote_jid,
              cs.CALL_INITIAL_AUDIO: True
            })

        before_events, after_events = disconnect_conn(q, conn, stream,
            [EventPattern('dbus-error', method='CreateChannel')])

        # CreateChannel should now return Disconnected
        assert before_events[0].error.get_dbus_name() == cs.DISCONNECTED, \
            before_events[0].error

if __name__ == '__main__':
    exec_test(lambda q, bus, conn, stream:
        test(q, bus, conn, stream, cs.CHANNEL_TYPE_CALL), timeout=10)
    exec_test(lambda q, bus, conn, stream:
        test(q, bus, conn, stream, cs.CHANNEL_TYPE_STREAMED_MEDIA), timeout=10)

<|endoftext|>


<commit_before>from __future__ import absolute_import
from __future__ import unicode_literals

import os

from django.test import SimpleTestCase

from corehq.apps.userreports.models import get_datasource_config
from corehq.form_processor.utils import convert_xform_to_json
from corehq.util.test_utils import TestFileMixin

BLACKLISTED_COLUMNS = ['received_on', 'inserted_at']

class TestPNCForms(SimpleTestCase, TestFileMixin):
    ucr_name = "static-icds-cas-static-postnatal_care_forms"
    domain = 'icds-cas'
    file_path = ('data', )
    root = os.path.dirname(__file__)
    maxDiff = None

    def test_ebf_form(self):
        config, _ = get_datasource_config(self.ucr_name, 'icds-cas')
        config.configured_indicators = [
            ind for ind in config.configured_indicators if ind['column_id'] != 'state_id'
        ]
        form_json = convert_xform_to_json(self.get_xml('ebf_form_v10326'))
        form_json = {
            'form': form_json,
            'domain': self.domain,
            'xmlns': form_json['@xmlns'],
            'doc_type': 'XFormInstance',
        }
        ucr_result = config.get_all_values(form_json)
        for row in ucr_result:
            row = {
                i.column.database_column_name: i.value
                for i in row
                if i.column.database_column_name not in BLACKLISTED_COLUMNS
            }

            self.assertEqual({
                "doc_id": None,
                "repeat_iteration": 0,
                "timeend": None,
                "ccs_record_case_id": "d53c940c-3bf3-44f7-97a1-f43fcbe74359",
                "child_health_case_id": "03f39da4-8ea3-4108-b8a8-1b58fdb4a698",
                "counsel_adequate_bf": None,
                "counsel_breast": None,
                "counsel_exclusive_bf": None,
                "counsel_increase_food_bf": None,
                "counsel_methods": None,
                "counsel_only_milk": 1,
                "skin_to_skin": None,
                "is_ebf": 1,
                "water_or_milk": 0,
                "other_milk_to_child": None,
                "tea_other": 0,
                "eating": 0,
                "not_breastfeeding": None
            }, row)

    def test_pnc_form(self):
        config, _ = get_datasource_config(self.ucr_name, 'icds-cas')
        config.configured_indicators = [
            ind for ind in config.configured_indicators if ind['column_id'] != 'state_id'
        ]
        form_json = convert_xform_to_json(self.get_xml('pnc_form_v10326'))
        form_json = {
            'form': form_json,
            'domain': self.domain,
            'xmlns': form_json['@xmlns'],
            'doc_type': 'XFormInstance',
        }
        ucr_result = config.get_all_values(form_json)
        for row in ucr_result:
            row = {
                i.column.database_column_name: i.value
                for i in row
                if i.column.database_column_name not in BLACKLISTED_COLUMNS
            }

            self.assertEqual({
                "doc_id": None,
                "repeat_iteration": 0,
                "timeend": None,
                "ccs_record_case_id": "081cc405-5598-430f-ac8f-39cc4a1fdb30",
                "child_health_case_id": "252d8e20-c698-4c94-a5a9-53bbf8972b64",
                "counsel_adequate_bf": None,
                "counsel_breast": None,
                "counsel_exclusive_bf": 1,
                "counsel_increase_food_bf": 1,
                "counsel_methods": None,
                "counsel_only_milk": None,
                "skin_to_skin": None,
                "is_ebf": 1,
                "water_or_milk": None,
                "other_milk_to_child": 0,
                "tea_other": None,
                "eating": None,
                "not_breastfeeding": None
            }, row)
<commit_msg>Patched get callcenter domains to return nothing.<commit_after>from __future__ import absolute_import
from __future__ import unicode_literals

import os

from django.test import SimpleTestCase
from mock import patch

from corehq.apps.userreports.models import get_datasource_config
from corehq.form_processor.utils import convert_xform_to_json
from corehq.util.test_utils import TestFileMixin

BLACKLISTED_COLUMNS = ['received_on', 'inserted_at']

@patch('corehq.apps.callcenter.data_source.get_call_center_domains', lambda: [])
class TestPNCForms(SimpleTestCase, TestFileMixin):
    ucr_name = "static-icds-cas-static-postnatal_care_forms"
    domain = 'icds-cas'
    file_path = ('data', )
    root = os.path.dirname(__file__)
    maxDiff = None

    def test_ebf_form(self):
        config, _ = get_datasource_config(self.ucr_name, 'icds-cas')
        config.configured_indicators = [
            ind for ind in config.configured_indicators if ind['column_id'] != 'state_id'
        ]
        form_json = convert_xform_to_json(self.get_xml('ebf_form_v10326'))
        form_json = {
            'form': form_json,
            'domain': self.domain,
            'xmlns': form_json['@xmlns'],
            'doc_type': 'XFormInstance',
        }
        ucr_result = config.get_all_values(form_json)
        for row in ucr_result:
            row = {
                i.column.database_column_name: i.value
                for i in row
                if i.column.database_column_name not in BLACKLISTED_COLUMNS
            }

            self.assertEqual({
                "doc_id": None,
                "repeat_iteration": 0,
                "timeend": None,
                "ccs_record_case_id": "d53c940c-3bf3-44f7-97a1-f43fcbe74359",
                "child_health_case_id": "03f39da4-8ea3-4108-b8a8-1b58fdb4a698",
                "counsel_adequate_bf": None,
                "counsel_breast": None,
                "counsel_exclusive_bf": None,
                "counsel_increase_food_bf": None,
                "counsel_methods": None,
                "counsel_only_milk": 1,
                "skin_to_skin": None,
                "is_ebf": 1,
                "water_or_milk": 0,
                "other_milk_to_child": None,
                "tea_other": 0,
                "eating": 0,
                "not_breastfeeding": None
            }, row)

    def test_pnc_form(self):
        config, _ = get_datasource_config(self.ucr_name, 'icds-cas')
        config.configured_indicators = [
            ind for ind in config.configured_indicators if ind['column_id'] != 'state_id'
        ]
        form_json = convert_xform_to_json(self.get_xml('pnc_form_v10326'))
        form_json = {
            'form': form_json,
            'domain': self.domain,
            'xmlns': form_json['@xmlns'],
            'doc_type': 'XFormInstance',
        }
        ucr_result = config.get_all_values(form_json)
        for row in ucr_result:
            row = {
                i.column.database_column_name: i.value
                for i in row
                if i.column.database_column_name not in BLACKLISTED_COLUMNS
            }

            self.assertEqual({
                "doc_id": None,
                "repeat_iteration": 0,
                "timeend": None,
                "ccs_record_case_id": "081cc405-5598-430f-ac8f-39cc4a1fdb30",
                "child_health_case_id": "252d8e20-c698-4c94-a5a9-53bbf8972b64",
                "counsel_adequate_bf": None,
                "counsel_breast": None,
                "counsel_exclusive_bf": 1,
                "counsel_increase_food_bf": 1,
                "counsel_methods": None,
                "counsel_only_milk": None,
                "skin_to_skin": None,
                "is_ebf": 1,
                "water_or_milk": None,
                "other_milk_to_child": 0,
                "tea_other": None,
                "eating": None,
                "not_breastfeeding": None
            }, row)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

"""
envoy.core
~~~~~~~~~~

This module provides
"""

import os
import shlex
import subprocess
import threading

__version__ = '0.0.0'
__license__ = 'MIT'
__author__ = 'Kenneth Reitz'

class Command(object):
    def __init__(self, cmd):
        self.cmd = cmd
        self.process = None
        self.out = None
        self.err = None
        self.returncode = None
        self.data = None

    def run(self, data, timeout):
        self.data = data
        def target():

            self.process = subprocess.Popen(self.cmd,
                universal_newlines=True,
                shell=False,
                env=os.environ,
                stdin=subprocess.PIPE,
                stdout=subprocess.PIPE,
                stderr=subprocess.PIPE,
                bufsize=0,
            )

            self.out, self.err = self.process.communicate(self.data)

        thread = threading.Thread(target=target)
        thread.start()

        thread.join(timeout)
        if thread.is_alive():
            self.process.terminate()
            thread.join()
        self.returncode = self.process.returncode
        return self.out, self.err

class Response(object):
    """A command's response"""

    def __init__(self, process=None):
        super(Response, self).__init__()

        self._process = process
        self.command = None
        self.std_err = None
        self.std_out = None
        self.status_code = None
        self.history = []

    def __repr__(self):
        if len(self.command):
            return '<Response [{0}]>'.format(self.command[0])
        else:
            return '<Response>'

def prep_args(command):
    """Parses command strings and returns a Popen-ready list."""

    # Prepare arguments.
    if isinstance(command, basestring):
        splitter = shlex.shlex(command, posix=True)
        splitter.whitespace = '|'
        splitter.whitespace_split = True
        command = []
        while True:
            token = splitter.get_token()
            if token:
                command.append(token)
            else:
                break

        command = map(shlex.split, command)

    return command

def run(command, data=None, timeout=None):
    """Executes a given commmand and returns Response.

    Blocks until process is complete, or timeout is reached.
    """

    command = prep_args(command)

    history = []
    for c in command:

        if len(history):
            # due to broken pipe problems pass only first 10MB
            data = history[-1].std_out[0:10*1024]

        cmd = Command(c)
        out, err = cmd.run(data, timeout)

        r = Response(process=cmd)

        r.command = c
        r.std_out = out
        r.std_err = err
        r.status_code = cmd.returncode

        history.append(r)

    r = history.pop()
    r.history = history

    return r
<commit_msg>Connected command stuff<commit_after># -*- coding: utf-8 -*-

"""
envoy.core
~~~~~~~~~~

This module provides
"""

import os
import shlex
import subprocess
import threading

__version__ = '0.0.0'
__license__ = 'MIT'
__author__ = 'Kenneth Reitz'

class Command(object):
    def __init__(self, cmd):
        self.cmd = cmd
        self.process = None
        self.out = None
        self.err = None
        self.returncode = None
        self.data = None

    def run(self, data, timeout):
        self.data = data
        def target():

            self.process = subprocess.Popen(self.cmd,
                universal_newlines=True,
                shell=False,
                env=os.environ,
                stdin=subprocess.PIPE,
                stdout=subprocess.PIPE,
                stderr=subprocess.PIPE,
                bufsize=0,
            )

            self.out, self.err = self.process.communicate(self.data)

        thread = threading.Thread(target=target)
        thread.start()

        thread.join(timeout)
        if thread.is_alive():
            self.process.terminate()
            thread.join()
        self.returncode = self.process.returncode
        return self.out, self.err

class ConnectedCommand(object):
    def __init__(self,
        process=None,
        std_in=None,
        std_out=None,
        std_err=None):

        self._process = process
        self.std_in = std_in
        self.std_out = std_out
        self.std_err = std_out

    @property
    def status_code(self):
        """The status code of the process.

        If the code is None, assume that it's still running.
        """
        return None

    @property
    def pid(self):
        """The process' PID."""
        return 0

    def kill(self):
        """Kills the process."""
        return True

    def expect(self, bytes, stream=None):
        """Block until given bytes appear in the stream."""
        if stream is None:
            stream = self.std_out
        pass

    def sendline(self, end='\n'):
        """Sends a line to std_in."""
        pass

    def block(self):
        """Blocks until command finishes. Returns Response instance."""
        pass

        self._process = process

        self._status_code = process.returncode
        self.std_in = process.stdin
        self.std_out = process.stdout
        self.std_err = process.stderr

    def __enter__(self):
        return self

    def __exit__(self, type, value, traceback):
        self.kill()

    @property
    def status_code(self):
        """
        The status code of the process.
        If the code is None, assume that it's still running.
        """
        if self._status_code is not None:
            return self._status_code
        # investigate
        return None

    @property
    def pid(self):
        """The process' PID."""
        return self._process.pid

    def kill(self):
        """Kills the process."""
        return self._process.kill()

    def expect(self, bytes, stream=None):
        """Block until given bytes appear in the stream."""
        if stream is None:
            stream = self.std_out
        pass

    def send(self, end='\n'):
        """Sends a line to std_in."""
        pass

    def block(self):
        """Blocks until command finishes. Returns Response instance."""
        self._status_code = self._process.wait()

class Response(object):
    """A command's response"""

    def __init__(self, process=None):
        super(Response, self).__init__()

        self._process = process
        self.command = None
        self.std_err = None
        self.std_out = None
        self.status_code = None
        self.history = []

    def __repr__(self):
        if len(self.command):
            return '<Response [{0}]>'.format(self.command[0])
        else:
            return '<Response>'

def expand_args(command):
    """Parses command strings and returns a Popen-ready list."""

    # Prepare arguments.
    if isinstance(command, basestring):
        splitter = shlex.shlex(command, posix=True)
        splitter.whitespace = '|'
        splitter.whitespace_split = True
        command = []
        while True:
            token = splitter.get_token()
            if token:
                command.append(token)
            else:
                break

        command = map(shlex.split, command)

    return command

def run(command, data=None, timeout=None):
    """Executes a given commmand and returns Response.

    Blocks until process is complete, or timeout is reached.
    """

    command = expand_args(command)

    history = []
    for c in command:

        if len(history):
            # due to broken pipe problems pass only first 10MB
            data = history[-1].std_out[0:10*1024]

        cmd = Command(c)
        out, err = cmd.run(data, timeout)

        r = Response(process=cmd)

        r.command = c
        r.std_out = out
        r.std_err = err
        r.status_code = cmd.returncode

        history.append(r)

    r = history.pop()
    r.history = history

    return r
def connect():
    pass
def connect(command, data=None):
    """Spawns a new process from the given command.
    """

    # TODO: support piped commands
    command_str = expand_args(command).pop()

    # cmd = ConnectedCommand()

    # def target():

    process = subprocess.Popen(command_str,
        universal_newlines=True,
        shell=False,
        env=os.environ,
        stdin=subprocess.PIPE,
        stdout=subprocess.PIPE,
        stderr=subprocess.PIPE,
        bufsize=0,
    )

    c = ConnectedCommand(process=process)
    return c
        # out, err = process.communicate(data)

    # thread = threading.Thread(target=target)
    # thread.start()

    # self.returncode = self.process.returncode

    return cmd
<|endoftext|>


<commit_before>#!/usr/bin/env python

# This is the refactored version for the benchmark script. Once this
# is completely under test coverage, the original benchmark.py gets
# turned into this, then this can be deleted.

# TODO: Why are records over & underclumped? ie. Charles Linder 7475467,
# Nirmala Ramanujam 7570988, Edwin L. Thomas 7799416

# TODO: Investigate "WEIRDNESS" output : Jaro-Winkler exception
# thrown on comparison between  and ROBERT BRUCE
# Underclumping: unique records vs. total records

import sqlite3, sys, csv, datetime;
import argparse
sys.path.append( './lib/' )
sys.path.append("lib")
from fwork import *
from bmconfig import *

# This is what is known as "testable code"
def get_filename_suffix(filename):
    return filename.split(".")[-1].lower()

# More testable code...
def is_csv_file(filename):
    return get_filename_suffix(filename) == "csv"

def print_diagnostics(data, table, header, tList):
   print "Printing diagnostics..."
   print "data: ", data
   print "table: ", table
   print "header: ", header
   print "tList: ", tList

def create_match_tables(c, fBnme, uqB, exCom, exAnd):
    # TODO: Split this query into two functions, test each
    c.executescript("""
        /* EXPAND UNIQUE BASE AND INDICATE ACTIVE MATCHES */
        CREATE TABLE dataM3 AS
        SELECT  uqS, a.*
          FROM (SELECT  uqS AS uqSUB, a.*
              FROM (SELECT  uqB, b.*
                  FROM  (SELECT DISTINCT(uqB) FROM dataM2 WHERE uqB!="") AS a
                INNER JOIN  %s AS b
                    ON  a.uqB=b.%s) AS a
             LEFT JOIN (SELECT %s, uqB, uqS FROM dataM2) AS b
                ON  a.uqB=b.uqB AND %s) AS a
        INNER JOIN (SELECT DISTINCT uqB, uqS FROM dataM2) AS b
            ON  a.%s=b.uqB;

        /* INDICATE INVENTORS WHO DO NOT MATCH */
        CREATE TABLE dataM4 AS
        SELECT  errD(a.ErrUQ, uqB) AS ErrUQ, b.*
          FROM (SELECT uqS, freqUQ(uqB) as ErrUQ FROM dataM3 GROUP BY uqS) AS a
        INNER JOIN  dataM3 AS b
            ON  a.uqS=b.uqS AND b.AppYear <= '2010' /*AND a.uqS not in (83, 85, 93)*/
          ORDER BY  uqS, %s;

        """ % (fBnme, uqB, exCom, exAnd, uqB, exCom))

def handle_fuzzy_dataS(c, exCom, uqB, uqS, fuzzy, fBnme, exAnd):
	# TODO: Remove leading CREATE INDEX as its already been created in the
	# calling function
	# TODO: Split this into 3 functions, no reason to do all this work in
	# one monster query.
        c.executescript("""
        CREATE INDEX IF NOT EXISTS dS_E ON dataS (%s);

        /* RETAIN ONLY JARO>0.9 FUZZY AND EXACT MATCHES */
        CREATE TABLE dataM AS
            SELECT  a.*, %s AS uqB, %s AS uqS, %s AS jaro
              FROM  %s AS a
        INNER JOIN  dataS AS b
            ON  %s
             WHERE  jaro>0.90;

        /* DETERMINE MAXIMUM JARO FOR EACH UQ AND EXACT COMBO */
        CREATE TABLE dataT AS
            SELECT  uqS, %s, MAX(jaro) AS jaro, count(*) as cnt
              FROM  dataM
          GROUP BY  uqS, %s;

        /* RETAIN ONLY MAXIMUM JARO */
        CREATE TABLE dataM2 AS
            SELECT  a.*
              FROM  dataM AS a
        INNER JOIN  dataT AS b
            ON  a.uqS=b.uqS AND a.jaro=b.jaro AND %s;
        """ % (exCom, uqB, uqS,
               "*".join(["jarow(a.%s, b.%s)" % (x,x) for x in fuzzy]),
               fBnme, exAnd, exCom, exCom, exAnd))

def handle_nonfuzzy_dataS(uqB, uqS, fBnme, exAnd):
    c.executescript("""
	CREATE TABLE dataM2 AS
	    SELECT  *, %s AS uqB, %s AS uqS
	      FROM  %s AS a
	INNER JOIN  dataS AS b
		ON  %s;
	""" % (uqB, uqS, fBnme, exAnd))

def export_csv_results(c, output):
	writer = csv.writer(open(output, "wb"), lineterminator="\n")
	writer.writerows([[x[1] for x in c.execute("PRAGMA TABLE_INFO(dataM4)")]])
	writer.writerows(c.execute("SELECT * FROM dataM4").fetchall())

# TODO: Create functions for handling true and false positives and negatives,
# compute all the relevant statistics using those measures instead of the
# mess of inline computation following the heredoc.
def print_results(c, output, t):
	#print "Printing results ..." + str(datetime.datetime.now())
	rep = [list(x) for x in c.execute("SELECT ErrUQ, uqSUB FROM dataM4")]
	#print "Rep: ", rep
	orig = len([x for x in rep if x[1]!=None])
	errm = sum([int(x[0]) for x in rep if x[0]!=None])
	#print errm
	u = 1.0*errm/orig
	o = 1-(float(orig)/len(rep))
	recall = 1.0 - u
	# overclumping is lumping
	# underclumping is splitting
	print """

	RESULTS ==================

	     Original: {original}
	  New Records: {new}
		Total: {total}

	    Overclump: {overclump} ({o:.2%})
	   Underclump: {underclump} ({u:.2%})
	    Precision: {precision:.2%}
	       Recall: {recall:.2%}
	  File Detail: {filename}
		 Time: {time}
	""".format(original = orig, new = len(rep)-orig, total = len(rep), overclump = len(rep)-orig, o = o,
		   underclump = errm, u = u, recall = recall, precision = recall/(recall+o), filename = output,
		   time = datetime.datetime.now()-t)

def attach_database(c, fileB, tblB, exCom, exAnd):
	# TODO: Replace with call to is_csv_file(fileB)
	if fileB.split(".")[-1].lower()=="csv":
	# TODO: Try to move some of this to a function
	    dataB = uniVert([x for x in csv.reader(open("%s" % fileB, "rb"))])

	    print_diagnostics(dataB, "dataB", True, ["Patent VARCHAR"])
	    #quickSQL(c, data=dataB,  table="dataB", header=True, typeList=["Patent VARCHAR"])
	    quickSQL2(c, data=dataB,  table="dataB", header=True, typeList=["Patent VARCHAR"])

	    c.execute("CREATE INDEX IF NOT EXISTS dB_E ON dataB (%s)" % exCom)
	    c.execute("CREATE INDEX IF NOT EXISTS dB_U ON dataB (%s)" % uqB)
	    fBnme = "dataB"
	# fileB is an SQLite3 database file...
	else:
	    c.execute("ATTACH DATABASE '%s' AS db" % fileB)
	    # fBnme is, apparently a table name. Or maybe a tbl nme...
	    if tblB=="":
		fBnme = "db.%s" % fileB.split(".")[-2].split("/")[-1]
	    else:
		fBnme = "db.%s" % tblB

        return fBnme

def handle_dataS(c, exCom, uqB, uqS, fuzzy, fBnme, exAnd):
	c.execute("CREATE INDEX IF NOT EXISTS dS_E ON dataS (%s);" % (exCom))
	if fuzzy:
	    handle_fuzzy_dataS(c, exCom, uqB, uqS, fuzzy, fBnme, exAnd)
	else:
	    handle_nonfuzzy_dataS(uqB, uqS, fBnme, exAnd)

def print_match_qualifiers(exact, fuzzy, uqS):
	print "Exact: ", exact
	print "Fuzzy: ", fuzzy
	print "uqS: ", uqS

def bmVerify(results, filepath="", outdir = ""):
        """
        Analysis function on disambiguation results, assuming that all benchmark data
        are in the large results dataset.

        Creates analysis detail csv file and prints summary information on
        over- and underclumping statistics.

        Running from the command line (make sure to set correct file paths in file)
        python bmVerify_v3.py "input filepath" "output directory" databases
        example:
        python bmVerify_v3.py /home/ysun/disambig/newcode/all/ /home/ayu/results_v2/ invpatC_NBNA.good.Jan2011 invpatC_NBYA.good.Jan2011 invpatC_YBNA.good.Jan2011

        Running interactively:
        import bmVerify_v3
        bmVerify(['final_r7', 'final_r8'], filepath="/home/ysun/disambig/newcode/all/", outdir = "/home/ayu/results_v2/")

        """

	# TODO: Move these out of this function completely, and into the top
	# part of this file.
	uqB = "Unique_Inventor_ID2"
	tblB = "invpat"
	#fileS = "/home/ron/disambig/BM/DefTruth5.csv"
	#fileS = "/home/doolin/src/patentbenchmarks/DefTruth5.csv"
	#fileS = "/home/doolin/src/patentbenchmarks/berkeley.csv"
	fileS = "/home/doolin/src/patentbenchmarks/pister.csv"
	#fileS = "/home/doolin/src/patentbenchmarks/paulrgray.csv"
	#fileS = "/home/doolin/src/patentbenchmarks/allbritton.csv"
	#fileS = "/home/doolin/src/patentbenchmarks/siddhu.csv"
	#fileS = "/var/share/patentdata/disambiguation/experiments/earth/berkeley/benchmark.csv"

        for result in results:

                fileB = filepath + "{result}.sqlite3".format(result=result)
                output = outdir + "{result}_DT5.csv".format(result=result)

                t = datetime.datetime.now()

                #print "Start time: " + str(datetime.datetime.now())

		# TODO: Move freqUQ out of this function if possible.
                class freqUQ:
                    def __init__(self):
                        self.list=[]
                    def step(self, value):
                        self.list.append(value)
                    def finalize(self):
                        return sorted([(self.list.count(x), x) for x in set(self.list)], reverse=True)[0][1]

                #MAKE THIS SO IT CAN ATTACH SQLITE3 FOR BENCHMARK
		# UniqueID,Patent,Lastname,Firstname
		# ,VARCHAR,,
		# ,%08d,,
		# UNIQUE,EXACT,FUZZY,FUZZY
		# 1,5773227,ALLBRITTON,NANCY L
		# TODO: Refactor into read_benchmark_csvfile()
		# This will help for unit test and provide self-documentation
                dataS = uniVert([x for x in csv.reader(open(fileS, "rb"))])
                #print dataS

		# TODO: Refactor into create_column_types()
                #1 = Variables, 2 = Type, 3 = Format (If necessary), 4 = Matching Type
                tList = ["%s %s" % (dataS[0][i], x) for i,x in enumerate(dataS[1]) if  x != ""]

		# Slice out rows 1, 2 & 3 from dataS. This is the data which gets
		# put in the database created below, and used for matching.
		# TODO: Refactor into slice_out_header()
                dataS2 = [dataS[0]]
                dataS2.extend(dataS[4:])

                # Format if its necessary --> Basically for Patents..
                for i,x in enumerate(dataS[2]):
                    if x!="":
                        for j in xrange(1,len(dataS2)):
                            if dataS2[j][i].isdigit():
                                dataS2[j][i] = x % int(dataS2[j][i])

                conn = sqlite3.connect(":memory:")
                #conn = sqlite3.connect("bmark2.sqlite3")
                conn.create_function("jarow", 2, jarow)
                conn.create_function("errD", 2, lambda x,y: (x!=y) and 1 or None)
                conn.create_aggregate("freqUQ", 1, freqUQ)
                c = conn.cursor()

                # FIGURE OUT WHICH ONES HAVE EXACT/FUZZY
                exact = [dataS[0][i] for i,x in enumerate(dataS[3]) if x.upper()[0]=="E"]
                fuzzy = [dataS[0][i] for i,x in enumerate(dataS[3]) if x.upper()[0]=="F"]
                uqS =   [dataS[0][i] for i,x in enumerate(dataS[3]) if x.upper()[0]=="U"][0]
                #print_match_qualifiers(exact, fuzzy, uqS)

                # CREATE INDEX, MERGE DATA BASED ON EXACTS
                #print "Creating indices... " + str(datetime.datetime.now())

                exAnd = " AND ".join(["a.%s=b.%s" % (x, x) for x in exact])
		#print "exAnd: ", exAnd
                exCom = ", ".join(exact)
                #print "exCom: ", exCom

                fBnme = attach_database(c, fileB, tblB, exCom, exAnd)

                #print_diagnostics(dataS2, "dataS", True, tList)
                #quickSQL(c, data=dataS2, table="dataS", header=True, typeList=tList)
                quickSQL2(c, data=dataS2, table="dataS", header=True, typeList=tList)

		handle_dataS(c, exCom, uqB, uqS, fuzzy, fBnme, exAnd)

                create_match_tables(c, fBnme, uqB, exCom, exAnd)

                #print "Indices Done ... " + str(datetime.datetime.now())

		export_csv_results(c, output)
		print_results(c, output, t)
                c.close()
                conn.close()

if __name__ == "__main__":
    import sys
    if(sys.argv[1] == 'help' or sys.argv[1] == '?'):
        print bmVerify.__doc__

    else:
            bmVerify(sys.argv[3:], sys.argv[1], sys.argv[2])
<commit_msg>added more todo items<commit_after>#!/usr/bin/env python

# This is the refactored version for the benchmark script. Once this
# is completely under test coverage, the original benchmark.py gets
# turned into this, then this can be deleted.

# TODO: Why are records over & underclumped? ie. Charles Linder 7475467,
# Nirmala Ramanujam 7570988, Edwin L. Thomas 7799416

# TODO: Investigate "WEIRDNESS" output : Jaro-Winkler exception
# thrown on comparison between  and ROBERT BRUCE
# Underclumping: unique records vs. total records

import sqlite3, sys, csv, datetime;
import argparse
sys.path.append( './lib/' )
sys.path.append("lib")
from fwork import *
from bmconfig import *

# This is what is known as "testable code"
def get_filename_suffix(filename):
    return filename.split(".")[-1].lower()

# More testable code...
def is_csv_file(filename):
    return get_filename_suffix(filename) == "csv"

def print_diagnostics(data, table, header, tList):
   print "Printing diagnostics..."
   print "data: ", data
   print "table: ", table
   print "header: ", header
   print "tList: ", tList

def create_match_tables(c, fBnme, uqB, exCom, exAnd):
    # TODO: Split this query into two functions, test each
    c.executescript("""
        /* EXPAND UNIQUE BASE AND INDICATE ACTIVE MATCHES */
        CREATE TABLE dataM3 AS
        SELECT  uqS, a.*
          FROM (SELECT  uqS AS uqSUB, a.*
              FROM (SELECT  uqB, b.*
                  FROM  (SELECT DISTINCT(uqB) FROM dataM2 WHERE uqB!="") AS a
                INNER JOIN  %s AS b
                    ON  a.uqB=b.%s) AS a
             LEFT JOIN (SELECT %s, uqB, uqS FROM dataM2) AS b
                ON  a.uqB=b.uqB AND %s) AS a
        INNER JOIN (SELECT DISTINCT uqB, uqS FROM dataM2) AS b
            ON  a.%s=b.uqB;

        /* INDICATE INVENTORS WHO DO NOT MATCH */
        CREATE TABLE dataM4 AS
        SELECT  errD(a.ErrUQ, uqB) AS ErrUQ, b.*
          FROM (SELECT uqS, freqUQ(uqB) as ErrUQ FROM dataM3 GROUP BY uqS) AS a
        INNER JOIN  dataM3 AS b
            ON  a.uqS=b.uqS AND b.AppYear <= '2010' /*AND a.uqS not in (83, 85, 93)*/
          ORDER BY  uqS, %s;

        """ % (fBnme, uqB, exCom, exAnd, uqB, exCom))

def handle_fuzzy_dataS(c, exCom, uqB, uqS, fuzzy, fBnme, exAnd):
	# TODO: Remove leading CREATE INDEX as its already been created in the
	# calling function
	# TODO: Split this into 3 functions, no reason to do all this work in
	# one monster query.
        c.executescript("""
        CREATE INDEX IF NOT EXISTS dS_E ON dataS (%s);

        /* RETAIN ONLY JARO>0.9 FUZZY AND EXACT MATCHES */
        CREATE TABLE dataM AS
            SELECT  a.*, %s AS uqB, %s AS uqS, %s AS jaro
              FROM  %s AS a
        INNER JOIN  dataS AS b
            ON  %s
             WHERE  jaro>0.90;

        /* DETERMINE MAXIMUM JARO FOR EACH UQ AND EXACT COMBO */
        CREATE TABLE dataT AS
            SELECT  uqS, %s, MAX(jaro) AS jaro, count(*) as cnt
              FROM  dataM
          GROUP BY  uqS, %s;

        /* RETAIN ONLY MAXIMUM JARO */
        CREATE TABLE dataM2 AS
            SELECT  a.*
              FROM  dataM AS a
        INNER JOIN  dataT AS b
            ON  a.uqS=b.uqS AND a.jaro=b.jaro AND %s;
        """ % (exCom, uqB, uqS,
               "*".join(["jarow(a.%s, b.%s)" % (x,x) for x in fuzzy]),
               fBnme, exAnd, exCom, exCom, exAnd))

def handle_nonfuzzy_dataS(uqB, uqS, fBnme, exAnd):
    c.executescript("""
	CREATE TABLE dataM2 AS
	    SELECT  *, %s AS uqB, %s AS uqS
	      FROM  %s AS a
	INNER JOIN  dataS AS b
		ON  %s;
	""" % (uqB, uqS, fBnme, exAnd))

def export_csv_results(c, output):
	writer = csv.writer(open(output, "wb"), lineterminator="\n")
	writer.writerows([[x[1] for x in c.execute("PRAGMA TABLE_INFO(dataM4)")]])
	writer.writerows(c.execute("SELECT * FROM dataM4").fetchall())

# TODO: Create functions for handling true and false positives and negatives,
# compute all the relevant statistics using those measures instead of the
# mess of inline computation following the heredoc.
# TODO: Switch to printing json results
def print_results(c, output, t):
	#print "Printing results ..." + str(datetime.datetime.now())
	rep = [list(x) for x in c.execute("SELECT ErrUQ, uqSUB FROM dataM4")]
	#print "Rep: ", rep
	orig = len([x for x in rep if x[1]!=None])
	errm = sum([int(x[0]) for x in rep if x[0]!=None])
	#print errm
	u = 1.0*errm/orig
	o = 1-(float(orig)/len(rep))
	recall = 1.0 - u
	# overclumping is lumping
	# underclumping is splitting
	print """

	RESULTS ==================

	     Original: {original}
	  New Records: {new}
		Total: {total}

	    Overclump: {overclump} ({o:.2%})
	   Underclump: {underclump} ({u:.2%})
	    Precision: {precision:.2%}
	       Recall: {recall:.2%}
	  File Detail: {filename}
		 Time: {time}
	""".format(original = orig, new = len(rep)-orig, total = len(rep), overclump = len(rep)-orig, o = o,
		   underclump = errm, u = u, recall = recall, precision = recall/(recall+o), filename = output,
		   time = datetime.datetime.now()-t)

def attach_database(c, fileB, tblB, exCom, exAnd):
	# TODO: Replace with call to is_csv_file(fileB),
	# after it's unit tested.
	if is_csv_file(fileB): #fileB.split(".")[-1].lower()=="csv":
	# TODO: Try to move some of this to a function
	    dataB = uniVert([x for x in csv.reader(open("%s" % fileB, "rb"))])

	    print_diagnostics(dataB, "dataB", True, ["Patent VARCHAR"])
	    #quickSQL(c, data=dataB,  table="dataB", header=True, typeList=["Patent VARCHAR"])
	    quickSQL2(c, data=dataB,  table="dataB", header=True, typeList=["Patent VARCHAR"])

	    c.execute("CREATE INDEX IF NOT EXISTS dB_E ON dataB (%s)" % exCom)
	    c.execute("CREATE INDEX IF NOT EXISTS dB_U ON dataB (%s)" % uqB)
	    fBnme = "dataB"
	# else assume fileB is an SQLite3 database file...
	else:
	    c.execute("ATTACH DATABASE '%s' AS db" % fileB)
	    # fBnme is, apparently a table name. Or maybe a tbl nme...
	    if tblB=="":
		fBnme = "db.%s" % fileB.split(".")[-2].split("/")[-1]
	    else:
		fBnme = "db.%s" % tblB

        return fBnme

def handle_dataS(c, exCom, uqB, uqS, fuzzy, fBnme, exAnd):
	c.execute("CREATE INDEX IF NOT EXISTS dS_E ON dataS (%s);" % (exCom))
	if fuzzy:
	    handle_fuzzy_dataS(c, exCom, uqB, uqS, fuzzy, fBnme, exAnd)
	else:
	    handle_nonfuzzy_dataS(uqB, uqS, fBnme, exAnd)

def print_match_qualifiers(exact, fuzzy, uqS):
	print "Exact: ", exact
	print "Fuzzy: ", fuzzy
	print "uqS: ", uqS

def bmVerify(results, filepath="", outdir = ""):
        """
        Analysis function on disambiguation results, assuming that all benchmark data
        are in the large results dataset.

        Creates analysis detail csv file and prints summary information on
        over- and underclumping statistics.

        Running from the command line (make sure to set correct file paths in file)
        python bmVerify_v3.py "input filepath" "output directory" databases
        example:
        python bmVerify_v3.py /home/ysun/disambig/newcode/all/ /home/ayu/results_v2/ invpatC_NBNA.good.Jan2011 invpatC_NBYA.good.Jan2011 invpatC_YBNA.good.Jan2011

        Running interactively:
        import bmVerify_v3
        bmVerify(['final_r7', 'final_r8'], filepath="/home/ysun/disambig/newcode/all/", outdir = "/home/ayu/results_v2/")

        """

	# TODO: Move these out of this function completely, and into the top
	# part of this file.
	uqB = "Unique_Inventor_ID2"
	tblB = "invpat"
	#fileS = "/home/ron/disambig/BM/DefTruth5.csv"
	#fileS = "/home/doolin/src/patentbenchmarks/DefTruth5.csv"
	#fileS = "/home/doolin/src/patentbenchmarks/berkeley.csv"
	fileS = "/home/doolin/src/patentbenchmarks/pister.csv"
	#fileS = "/home/doolin/src/patentbenchmarks/paulrgray.csv"
	#fileS = "/home/doolin/src/patentbenchmarks/allbritton.csv"
	#fileS = "/home/doolin/src/patentbenchmarks/siddhu.csv"
	#fileS = "/var/share/patentdata/disambiguation/experiments/earth/berkeley/benchmark.csv"

        for result in results:

                fileB = filepath + "{result}.sqlite3".format(result=result)
                output = outdir + "{result}_DT5.csv".format(result=result)

                t = datetime.datetime.now()

                #print "Start time: " + str(datetime.datetime.now())

		# TODO: Move freqUQ out of this function if possible.
                class freqUQ:
                    def __init__(self):
                        self.list=[]
                    def step(self, value):
                        self.list.append(value)
                    def finalize(self):
                        return sorted([(self.list.count(x), x) for x in set(self.list)], reverse=True)[0][1]

                #MAKE THIS SO IT CAN ATTACH SQLITE3 FOR BENCHMARK
		# UniqueID,Patent,Lastname,Firstname
		# ,VARCHAR,,
		# ,%08d,,
		# UNIQUE,EXACT,FUZZY,FUZZY
		# 1,5773227,ALLBRITTON,NANCY L
		# TODO: Refactor into read_benchmark_csvfile()
		# This will help for unit test and provide self-documentation
                dataS = uniVert([x for x in csv.reader(open(fileS, "rb"))])
                #print dataS

		# TODO: Refactor into create_column_types()
                #1 = Variables, 2 = Type, 3 = Format (If necessary), 4 = Matching Type
                tList = ["%s %s" % (dataS[0][i], x) for i,x in enumerate(dataS[1]) if  x != ""]

		# Slice out rows 1, 2 & 3 from dataS. This is the data which gets
		# put in the database created below, and used for matching.
		# TODO: Refactor into slice_header_from()
                dataS2 = [dataS[0]]
                dataS2.extend(dataS[4:])

                # Format if its necessary --> Basically for Patents..
                for i,x in enumerate(dataS[2]):
                    if x!="":
                        for j in xrange(1,len(dataS2)):
                            if dataS2[j][i].isdigit():
                                dataS2[j][i] = x % int(dataS2[j][i])

                conn = sqlite3.connect(":memory:")
                #conn = sqlite3.connect("bmark2.sqlite3")
                conn.create_function("jarow", 2, jarow)
                conn.create_function("errD", 2, lambda x,y: (x!=y) and 1 or None)
                conn.create_aggregate("freqUQ", 1, freqUQ)
                c = conn.cursor()

                # FIGURE OUT WHICH ONES HAVE EXACT/FUZZY
                exact = [dataS[0][i] for i,x in enumerate(dataS[3]) if x.upper()[0]=="E"]
                fuzzy = [dataS[0][i] for i,x in enumerate(dataS[3]) if x.upper()[0]=="F"]
                uqS =   [dataS[0][i] for i,x in enumerate(dataS[3]) if x.upper()[0]=="U"][0]
                #print_match_qualifiers(exact, fuzzy, uqS)

                # CREATE INDEX, MERGE DATA BASED ON EXACTS
                #print "Creating indices... " + str(datetime.datetime.now())

                exAnd = " AND ".join(["a.%s=b.%s" % (x, x) for x in exact])
		#print "exAnd: ", exAnd
                exCom = ", ".join(exact)
                #print "exCom: ", exCom

                fBnme = attach_database(c, fileB, tblB, exCom, exAnd)

                #print_diagnostics(dataS2, "dataS", True, tList)
                #quickSQL(c, data=dataS2, table="dataS", header=True, typeList=tList)
                quickSQL2(c, data=dataS2, table="dataS", header=True, typeList=tList)

		handle_dataS(c, exCom, uqB, uqS, fuzzy, fBnme, exAnd)

                create_match_tables(c, fBnme, uqB, exCom, exAnd)

                #print "Indices Done ... " + str(datetime.datetime.now())

		export_csv_results(c, output)
		print_results(c, output, t)
                c.close()
                conn.close()

if __name__ == "__main__":
    import sys
    if(sys.argv[1] == 'help' or sys.argv[1] == '?'):
        print bmVerify.__doc__

    else:
            bmVerify(sys.argv[3:], sys.argv[1], sys.argv[2])
<|endoftext|>


<commit_before>import click
import json
import importlib
import os
import sys

from shakedown.cli.helpers import *

@click.command('shakedown')
@click.argument('tests', nargs=-1)
@click.option('-u', '--dcos-url', help='URL to a running DC/OS cluster.')
@click.option('-f', '--fail', type=click.Choice(['fast', 'never']), help='Sepcify whether to continue testing when encountering failures. (default: fast)')
@click.option('-i', '--ssh-key-file', type=click.Path(), help='Path to the SSH keyfile to use for authentication.')
@click.option('-q', '--quiet', is_flag=True, help='Suppress all superfluous output.')
@click.option('-k', '--ssl-no-verify', is_flag=True, help='Suppress SSL certificate verification.')
@click.option('-o', '--stdout', type=click.Choice(['pass', 'fail', 'skip', 'all', 'none']), help='Print the standard output of tests with the specified result. (default: fail)')
@click.option('-s', '--stdout-inline', is_flag=True, help='Display output inline rather than after test phase completion.')
@click.option('-p', '--pytest-option', multiple=True, help='Options flags to pass to pytest.')
@click.option('-t', '--oauth-token', help='OAuth token to use for DC/OS authentication.')
@click.option('-n', '--username', help='Username to use for DC/OS authentication.')
@click.option('-w', '--password', hide_input=True, help='Password to use for DC/OS authentication.')
@click.option('--no-banner', is_flag=True, help='Suppress the product banner.')
@click.version_option(version=shakedown.VERSION)

def cli(**args):
    """ Shakedown is a DC/OS test-harness wrapper for the pytest tool.
    """
    import shakedown

    # Read configuration options from ~/.shakedown (if exists)
    args = read_config(args)

    # Set configuration defaults
    args = set_config_defaults(args)

    if args['quiet']:
        shakedown.cli.quiet = True

    if not args['dcos_url']:
        click.secho('error: --dcos-url is a required option; see --help for more information.', fg='red', bold=True)
        sys.exit(1)

    if args['ssh_key_file']:
        shakedown.cli.ssh_key_file = args['ssh_key_file']

    if not args['no_banner']:
        echo(banner(), n=False)

    echo('Running pre-flight checks...', d='step-maj')

    # required modules and their 'version' method
    imported = {}
    requirements = {
        'pytest': '__version__',
        'dcos': 'version'
    }

    for req in requirements:
        ver = requirements[req]

        echo("Checking for {} library...".format(req), d='step-min', n=False)
        try:
            imported[req] = importlib.import_module(req, package=None)
        except ImportError:
            click.secho("error: {p} is not installed; run 'pip install {p}'.".format(p=req), fg='red', bold=True)
            sys.exit(1)

        echo(getattr(imported[req], requirements[req]))

    if args['ssl_no_verify']:
        imported['dcos'].config.set_val('core.ssl_verify', 'False')

    echo('Checking for DC/OS cluster...', d='step-min', n=False)

    with stdchannel_redirected(sys.stderr, os.devnull):
        imported['dcos'].config.set_val('core.dcos_url', args['dcos_url'])

    try:
        echo(shakedown.dcos_version())
    except:
        click.secho("error: cluster '" + args['dcos_url'] + "' is unreachable.", fg='red', bold=True)
        sys.exit(1)

    echo('Authenticating with cluster...', d='step-maj')
    authenticated = False
    token = imported['dcos'].config.get_config_val("core.dcos_acs_token")
    if token is not None:
        echo('Validating existing ACS token...', d='step-min', n=False)
        try:
            shakedown.dcos_leader()

            echo('ok')
            authenticated = True
        except imported['dcos'].errors.DCOSException:
            click.secho("error: authentication failed.", fg='red', bold=True)
    if not authenticated and args['oauth_token']:
       try:
            echo('Validating OAuth token...', d='step-min', n=False)
            token = shakedown.authenticate_oauth(args['oauth_token'])

            with stdchannel_redirected(sys.stderr, os.devnull):
                imported['dcos'].config.set_val('core.dcos_acs_token', token)

            authenticated = True
            echo('ok')
       except:
            click.secho("error: authentication failed.", fg='red', bold=True)
    if not authenticated and args['username'] and args['password']:
        try:
            echo('Validating username and password...', d='step-min', n=False)
            token = shakedown.authenticate(args['username'], args['password'])

            with stdchannel_redirected(sys.stderr, os.devnull):
                imported['dcos'].config.set_val('core.dcos_acs_token', token)

            authenticated = True
            echo('ok')
        except:
            click.secho("error: authentication failed.", fg='red', bold=True)
    if not authenticated:
        click.secho("error: no authentication credentials or token found.", fg='red', bold=True)
        sys.exit(1)

    class shakedown:
        """ This encapsulates a PyTest wrapper plugin
        """

        state = {}

        stdout = []

        tests = {
            'file': {},
            'test': {}
        }

        report_stats = {
            'passed':[],
            'skipped':[],
            'failed':[],
            'total_passed':0,
            'total_skipped':0,
            'total_failed':0,
        }

        def output(title, state, text, status=True):
            """ Capture and display stdout/stderr output

                :param title: the title of the output box (eg. test name)
                :type title: str
                :param state: state of the result (pass, fail)
                :type state: str
                :param text: the stdout/stderr output
                :type text: str
                :param status: whether to output a status marker
                :type status: bool
            """
            if status:
                if state == 'fail':
                    echo(fchr('FF'), d='fail')
                elif state == 'pass':
                    echo(fchr('PP'), d='pass')

            if text and args['stdout'] in [state, 'all']:
                o = decorate('Output during ', 'quote-head-' + state)
                o += click.style(decorate(title, style=state), bold=True) + "\n"
                o += decorate(str(text).strip(), style='quote-' + state)

                if args['stdout_inline']:
                    echo(o)
                    if state == 'pass':
                        echo('')
                else:
                    shakedown.stdout.append(o)

        def pytest_collectreport(self, report):
            """ Collect and validate individual test files
            """

            if not 'collect' in shakedown.state:
                shakedown.state['collect'] = 1
                echo('Collecting and validating test files...', d='step-min')

            if report.nodeid:
                echo(report.nodeid, d='item-maj', n=False)

                state = None

                if report.failed:
                    state = 'fail'
                if report.passed:
                    state = 'pass'
                if report.skipped:
                    state = 'skip'

                if state:
                    if report.longrepr:
                        shakedown.output(report.nodeid, state, report.longrepr)
                    else:
                        shakedown.output(report.nodeid, state, None)

        def pytest_sessionstart(self):
            """ Tests have been collected, begin running them...
            """

            echo('Initiating testing phase...', d='step-maj')

        def pytest_report_teststatus(self, report):
            """ Print report results to the console as they are run
            """

            try:
                report_file, report_test = report.nodeid.split('::', 1)
            except ValueError:
                return

            if not 'test' in shakedown.state:
                shakedown.state['test'] = 1
                echo('Running individual tests...', d='step-min')

            if not report_file in shakedown.tests['file']:
                shakedown.tests['file'][report_file] = 1
                echo(report_file, d='item-maj')
            if not report.nodeid in shakedown.tests['test']:
                shakedown.tests['test'][report.nodeid] = {}
                echo(report_test, d='item-min', n=False)

            if report.failed:
                shakedown.tests['test'][report.nodeid]['fail'] = True

            if report.when == 'teardown' and not 'tested' in shakedown.tests['test'][report.nodeid]:
                shakedown.output(report.nodeid, 'pass', None)

            # Suppress excess terminal output
            return report.outcome, None, None

        def pytest_runtest_logreport(self, report):
            """ Log the [stdout, stderr] results of tests if desired
            """

            state = None

            for secname, content in report.sections:
                if report.failed:
                    state = 'fail'
                if report.passed:
                    state = 'pass'
                if report.skipped:
                    state = 'skip'

                if state and secname != 'Captured stdout call':
                    if not 'setup' in shakedown.tests['test'][report.nodeid]:
                        module = report.nodeid.split('::', 1)[0]
                        cap_type = secname.split(' ')[-1]
                        shakedown.tests['test'][report.nodeid]['setup'] = True
                        shakedown.output(module + ' ' + cap_type, state, content, False)
                elif state and report.when == 'call':
                    if 'tested' in shakedown.tests['test'][report.nodeid]:
                        shakedown.output(report.nodeid, state, content, False)
                    else:
                        shakedown.tests['test'][report.nodeid]['tested'] = True
                        shakedown.output(report.nodeid, state, content)

            # Capture execution crashes
            if hasattr(report.longrepr, 'reprcrash'):
                longreport = report.longrepr

                if 'tested' in shakedown.tests['test'][report.nodeid]:
                    shakedown.output(report.nodeid, 'fail', 'error: ' + str(longreport.reprcrash), False)

                    if args['stdout_inline']:
                        echo('')
                else:
                    shakedown.tests['test'][report.nodeid]['tested'] = True
                    shakedown.output(report.nodeid, 'fail', 'error: ' + str(longreport.reprcrash))

        def pytest_sessionfinish(self, session, exitstatus):
            """ Testing phase is complete; print extra reports (stdout/stderr, JSON) as requested
            """

            echo('Test phase completed.', d='step-maj')

            if ('stdout' in args and args['stdout']) and shakedown.stdout:
                for output in shakedown.stdout:
                    echo(output)

    opts = ['-q', '--tb=no']

    if args['fail'] == 'fast':
        opts.append('-x')

    if args['pytest_option']:
        for opt in args['pytest_option']:
            opts.append(opt)

    if args['tests']:
        tests_to_run = []
        for test in args['tests']:
            tests_to_run.extend(test.split())
        for test in tests_to_run:
            opts.append(test)

    exitstatus = imported['pytest'].main(opts, plugins=[shakedown()])

    sys.exit(exitstatus)
<commit_msg>(DCOS-469) Fix bug where `teardown` doesn't display<commit_after>import click
import json
import importlib
import os
import sys

from shakedown.cli.helpers import *

@click.command('shakedown')
@click.argument('tests', nargs=-1)
@click.option('-u', '--dcos-url', help='URL to a running DC/OS cluster.')
@click.option('-f', '--fail', type=click.Choice(['fast', 'never']), help='Sepcify whether to continue testing when encountering failures. (default: fast)')
@click.option('-i', '--ssh-key-file', type=click.Path(), help='Path to the SSH keyfile to use for authentication.')
@click.option('-q', '--quiet', is_flag=True, help='Suppress all superfluous output.')
@click.option('-k', '--ssl-no-verify', is_flag=True, help='Suppress SSL certificate verification.')
@click.option('-o', '--stdout', type=click.Choice(['pass', 'fail', 'skip', 'all', 'none']), help='Print the standard output of tests with the specified result. (default: fail)')
@click.option('-s', '--stdout-inline', is_flag=True, help='Display output inline rather than after test phase completion.')
@click.option('-p', '--pytest-option', multiple=True, help='Options flags to pass to pytest.')
@click.option('-t', '--oauth-token', help='OAuth token to use for DC/OS authentication.')
@click.option('-n', '--username', help='Username to use for DC/OS authentication.')
@click.option('-w', '--password', hide_input=True, help='Password to use for DC/OS authentication.')
@click.option('--no-banner', is_flag=True, help='Suppress the product banner.')
@click.version_option(version=shakedown.VERSION)

def cli(**args):
    """ Shakedown is a DC/OS test-harness wrapper for the pytest tool.
    """
    import shakedown

    # Read configuration options from ~/.shakedown (if exists)
    args = read_config(args)

    # Set configuration defaults
    args = set_config_defaults(args)

    if args['quiet']:
        shakedown.cli.quiet = True

    if not args['dcos_url']:
        click.secho('error: --dcos-url is a required option; see --help for more information.', fg='red', bold=True)
        sys.exit(1)

    if args['ssh_key_file']:
        shakedown.cli.ssh_key_file = args['ssh_key_file']

    if not args['no_banner']:
        echo(banner(), n=False)

    echo('Running pre-flight checks...', d='step-maj')

    # required modules and their 'version' method
    imported = {}
    requirements = {
        'pytest': '__version__',
        'dcos': 'version'
    }

    for req in requirements:
        ver = requirements[req]

        echo("Checking for {} library...".format(req), d='step-min', n=False)
        try:
            imported[req] = importlib.import_module(req, package=None)
        except ImportError:
            click.secho("error: {p} is not installed; run 'pip install {p}'.".format(p=req), fg='red', bold=True)
            sys.exit(1)

        echo(getattr(imported[req], requirements[req]))

    if args['ssl_no_verify']:
        imported['dcos'].config.set_val('core.ssl_verify', 'False')

    echo('Checking for DC/OS cluster...', d='step-min', n=False)

    with stdchannel_redirected(sys.stderr, os.devnull):
        imported['dcos'].config.set_val('core.dcos_url', args['dcos_url'])

    try:
        echo(shakedown.dcos_version())
    except:
        click.secho("error: cluster '" + args['dcos_url'] + "' is unreachable.", fg='red', bold=True)
        sys.exit(1)

    echo('Authenticating with cluster...', d='step-maj')
    authenticated = False
    token = imported['dcos'].config.get_config_val("core.dcos_acs_token")
    if token is not None:
        echo('Validating existing ACS token...', d='step-min', n=False)
        try:
            shakedown.dcos_leader()

            echo('ok')
            authenticated = True
        except imported['dcos'].errors.DCOSException:
            click.secho("error: authentication failed.", fg='red', bold=True)
    if not authenticated and args['oauth_token']:
       try:
            echo('Validating OAuth token...', d='step-min', n=False)
            token = shakedown.authenticate_oauth(args['oauth_token'])

            with stdchannel_redirected(sys.stderr, os.devnull):
                imported['dcos'].config.set_val('core.dcos_acs_token', token)

            authenticated = True
            echo('ok')
       except:
            click.secho("error: authentication failed.", fg='red', bold=True)
    if not authenticated and args['username'] and args['password']:
        try:
            echo('Validating username and password...', d='step-min', n=False)
            token = shakedown.authenticate(args['username'], args['password'])

            with stdchannel_redirected(sys.stderr, os.devnull):
                imported['dcos'].config.set_val('core.dcos_acs_token', token)

            authenticated = True
            echo('ok')
        except:
            click.secho("error: authentication failed.", fg='red', bold=True)
    if not authenticated:
        click.secho("error: no authentication credentials or token found.", fg='red', bold=True)
        sys.exit(1)

    class shakedown:
        """ This encapsulates a PyTest wrapper plugin
        """

        state = {}

        stdout = []

        tests = {
            'file': {},
            'test': {}
        }

        report_stats = {
            'passed':[],
            'skipped':[],
            'failed':[],
            'total_passed':0,
            'total_skipped':0,
            'total_failed':0,
        }

        def output(title, state, text, status=True):
            """ Capture and display stdout/stderr output

                :param title: the title of the output box (eg. test name)
                :type title: str
                :param state: state of the result (pass, fail)
                :type state: str
                :param text: the stdout/stderr output
                :type text: str
                :param status: whether to output a status marker
                :type status: bool
            """
            if status:
                if state == 'fail':
                    echo(fchr('FF'), d='fail')
                elif state == 'pass':
                    echo(fchr('PP'), d='pass')

            if text and args['stdout'] in [state, 'all']:
                o = decorate('Output during ', 'quote-head-' + state)
                o += click.style(decorate(title, style=state), bold=True) + "\n"
                o += decorate(str(text).strip(), style='quote-' + state)

                if args['stdout_inline']:
                    echo(o)
                    if state == 'pass':
                        echo('')
                else:
                    shakedown.stdout.append(o)

        def pytest_collectreport(self, report):
            """ Collect and validate individual test files
            """

            if not 'collect' in shakedown.state:
                shakedown.state['collect'] = 1
                echo('Collecting and validating test files...', d='step-min')

            if report.nodeid:
                echo(report.nodeid, d='item-maj', n=False)

                state = None

                if report.failed:
                    state = 'fail'
                if report.passed:
                    state = 'pass'
                if report.skipped:
                    state = 'skip'

                if state:
                    if report.longrepr:
                        shakedown.output(report.nodeid, state, report.longrepr)
                    else:
                        shakedown.output(report.nodeid, state, None)

        def pytest_sessionstart(self):
            """ Tests have been collected, begin running them...
            """

            echo('Initiating testing phase...', d='step-maj')

        def pytest_report_teststatus(self, report):
            """ Print report results to the console as they are run
            """

            try:
                report_file, report_test = report.nodeid.split('::', 1)
            except ValueError:
                return

            if not 'test' in shakedown.state:
                shakedown.state['test'] = 1
                echo('Running individual tests...', d='step-min')

            if not report_file in shakedown.tests['file']:
                shakedown.tests['file'][report_file] = 1
                echo(report_file, d='item-maj')
            if not report.nodeid in shakedown.tests['test']:
                shakedown.tests['test'][report.nodeid] = {}
                echo(report_test, d='item-min', n=False)

            if report.failed:
                shakedown.tests['test'][report.nodeid]['fail'] = True

            if report.when == 'teardown' and not 'tested' in shakedown.tests['test'][report.nodeid]:
                shakedown.output(report.nodeid, 'pass', None)

            # Suppress excess terminal output
            return report.outcome, None, None

        def pytest_runtest_logreport(self, report):
            """ Log the [stdout, stderr] results of tests if desired
            """

            state = None

            for secname, content in report.sections:
                if report.failed:
                    state = 'fail'
                if report.passed:
                    state = 'pass'
                if report.skipped:
                    state = 'skip'

                if state and secname != 'Captured stdout call':
                    module = report.nodeid.split('::', 1)[0]
                    cap_type = secname.split(' ')[-1]

                    if not 'setup' in shakedown.tests['test'][report.nodeid]:
                        shakedown.tests['test'][report.nodeid]['setup'] = True
                        shakedown.output(module + ' ' + cap_type, state, content, False)
                    elif cap_type == 'teardown':
                        shakedown.output(module + ' ' + cap_type, state, content, False)
                elif state and report.when == 'call':
                    if 'tested' in shakedown.tests['test'][report.nodeid]:
                        shakedown.output(report.nodeid, state, content, False)
                    else:
                        shakedown.tests['test'][report.nodeid]['tested'] = True
                        shakedown.output(report.nodeid, state, content)

            # Capture execution crashes
            if hasattr(report.longrepr, 'reprcrash'):
                longreport = report.longrepr

                if 'tested' in shakedown.tests['test'][report.nodeid]:
                    shakedown.output(report.nodeid, 'fail', 'error: ' + str(longreport.reprcrash), False)

                    if args['stdout_inline']:
                        echo('')
                else:
                    shakedown.tests['test'][report.nodeid]['tested'] = True
                    shakedown.output(report.nodeid, 'fail', 'error: ' + str(longreport.reprcrash))

        def pytest_sessionfinish(self, session, exitstatus):
            """ Testing phase is complete; print extra reports (stdout/stderr, JSON) as requested
            """

            echo('Test phase completed.', d='step-maj')

            if ('stdout' in args and args['stdout']) and shakedown.stdout:
                for output in shakedown.stdout:
                    echo(output)

    opts = ['-q', '--tb=no']

    if args['fail'] == 'fast':
        opts.append('-x')

    if args['pytest_option']:
        for opt in args['pytest_option']:
            opts.append(opt)

    if args['tests']:
        tests_to_run = []
        for test in args['tests']:
            tests_to_run.extend(test.split())
        for test in tests_to_run:
            opts.append(test)

    exitstatus = imported['pytest'].main(opts, plugins=[shakedown()])

    sys.exit(exitstatus)
<|endoftext|>


<commit_before>'''
bmp180 is a micropython module for the Bosch BMP180 sensor. It measures
temperature as well as pressure, with a high enough resolution to calculate
altitude.
Breakoutboard: http://www.adafruit.com/products/1603
data-sheet: http://ae-bst.resource.bosch.com/media/products/dokumente/
bmp180/BST-BMP180-DS000-09.pdf

The MIT License (MIT)
Copyright (c) 2014 Sebastian Plamauer, oeplse@gmail.com
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
'''

from ustruct import unpack as unp
from machine import I2C, Pin
import math
import time

# BMP180 class
class BMP180():
    '''
    Module for the BMP180 pressure sensor.
    '''

    _bmp_addr = 119             # adress of BMP180 is hardcoded on the sensor

    # init
    def __init__(self, i2c_bus):

        # create i2c obect
        _bmp_addr = self._bmp_addr
        self._bmp_i2c = i2c_bus
        self._bmp_i2c.start()
        self.chip_id = self._bmp_i2c.readfrom_mem(_bmp_addr, 0xD0, 2)
        # read calibration data from EEPROM
        self._AC1 = unp('>h', self._bmp_i2c.readfrom_mem(_bmp_addr, 0xAA, 2))[0]
        self._AC2 = unp('>h', self._bmp_i2c.readfrom_mem(_bmp_addr, 0xAC, 2))[0]
        self._AC3 = unp('>h', self._bmp_i2c.readfrom_mem(_bmp_addr, 0xAE, 2))[0]
        self._AC4 = unp('>H', self._bmp_i2c.readfrom_mem(_bmp_addr, 0xB0, 2))[0]
        self._AC5 = unp('>H', self._bmp_i2c.readfrom_mem(_bmp_addr, 0xB2, 2))[0]
        self._AC6 = unp('>H', self._bmp_i2c.readfrom_mem(_bmp_addr, 0xB4, 2))[0]
        self._B1 = unp('>h', self._bmp_i2c.readfrom_mem(_bmp_addr, 0xB6, 2))[0]
        self._B2 = unp('>h', self._bmp_i2c.readfrom_mem(_bmp_addr, 0xB8, 2))[0]
        self._MB = unp('>h', self._bmp_i2c.readfrom_mem(_bmp_addr, 0xBA, 2))[0]
        self._MC = unp('>h', self._bmp_i2c.readfrom_mem(_bmp_addr, 0xBC, 2))[0]
        self._MD = unp('>h', self._bmp_i2c.readfrom_mem(_bmp_addr, 0xBE, 2))[0]

        # settings to be adjusted by user
        self.oversample_setting = 3
        self.baseline = 101325.0

        # output raw
        self.UT_raw = None
        self.B5_raw = None
        self.MSB_raw = None
        self.LSB_raw = None
        self.XLSB_raw = None
        self.gauge = self.makegauge() # Generator instance
        for _ in range(128):
            next(self.gauge)
            time.sleep_ms(1)

    def compvaldump(self):
        '''
        Returns a list of all compensation values
        '''
        return [self._AC1, self._AC2, self._AC3, self._AC4, self._AC5, self._AC6,
                self._B1, self._B2, self._MB, self._MC, self._MD, self.oversample_setting]

    # gauge raw
    def makegauge(self):
        '''
        Generator refreshing the raw measurments.
        '''
        delays = (5, 8, 14, 25)
        while True:
            self._bmp_i2c.writeto_mem(self._bmp_addr, 0xF4, bytearray([0x2E]))
            t_start = time.ticks_ms()
            while (time.ticks_ms() - t_start) <= 5: # 5mS delay
                yield None
            try:
                self.UT_raw = self._bmp_i2c.readfrom_mem(self._bmp_addr, 0xF6, 2)
            except:
                yield None
            self._bmp_i2c.writeto_mem(self._bmp_addr, 0xF4, bytearray([0x34+(self.oversample_setting << 6)]))
            t_pressure_ready = delays[self.oversample_setting]
            t_start = time.ticks_ms()
            while (time.ticks_ms() - t_start) <= t_pressure_ready:
                yield None
            try:
                self.MSB_raw = self._bmp_i2c.readfrom_mem(self._bmp_addr, 0xF6, 1)
                self.LSB_raw = self._bmp_i2c.readfrom_mem(self._bmp_addr, 0xF7, 1)
                self.XLSB_raw = self._bmp_i2c.readfrom_mem(self._bmp_addr, 0xF8, 1)
            except:
                yield None
            yield True

    def blocking_read(self):
        if next(self.gauge) is not None: # Discard old data
            pass
        while next(self.gauge) is None:
            pass

    @property
    def oversample_sett(self):
        return self.oversample_setting

    @oversample_sett.setter
    def oversample_sett(self, value):
        if value in range(4):
            self.oversample_setting = value
        else:
            print('oversample_sett can only be 0, 1, 2 or 3, using 3 instead')
            self.oversample_setting = 3

    @property
    def temperature(self):
        '''
        Temperature in degree C.
        '''
        next(self.gauge)
        try:
            UT = unp('>h', self.UT_raw)[0]
        except:
            return 0.0
        X1 = (UT-self._AC6)*self._AC5/2**15
        X2 = self._MC*2**11/(X1+self._MD)
        self.B5_raw = X1+X2
        return (((X1+X2)+8)/2**4)/10

    @property
    def pressure(self):
        '''
        Pressure in mbar.
        '''
        next(self.gauge)
        self.temperature  # Populate self.B5_raw
        try:
            MSB = unp('B', self.MSB_raw)[0]
            LSB = unp('B', self.LSB_raw)[0]
            XLSB = unp('B', self.XLSB_raw)[0]
        except:
            return 0.0
        UP = ((MSB << 16)+(LSB << 8)+XLSB) >> (8-self.oversample_setting)
        B6 = self.B5_raw-4000
        X1 = (self._B2*(B6**2/2**12))/2**11
        X2 = self._AC2*B6/2**11
        X3 = X1+X2
        B3 = ((int((self._AC1*4+X3)) << self.oversample_setting)+2)/4
        X1 = self._AC3*B6/2**13
        X2 = (self._B1*(B6**2/2**12))/2**16
        X3 = ((X1+X2)+2)/2**2
        B4 = abs(self._AC4)*(X3+32768)/2**15
        B7 = (abs(UP)-B3) * (50000 >> self.oversample_setting)
        if B7 < 0x80000000:
            pressure = (B7*2)/B4
        else:
            pressure = (B7/B4)*2
        X1 = (pressure/2**8)**2
        X1 = (X1*3038)/2**16
        X2 = (-7357*pressure)/2**16
        return pressure+(X1+X2+3791)/2**4

    @property
    def altitude(self):
        '''
        Altitude in m.
        '''
        try:
            p = -7990.0*math.log(self.pressure/self.baseline)
        except:
            p = 0.0
        return p
<commit_msg>parse UT_raw as unsigned short, fixes temps > ~29.5°C<commit_after>'''
bmp180 is a micropython module for the Bosch BMP180 sensor. It measures
temperature as well as pressure, with a high enough resolution to calculate
altitude.
Breakoutboard: http://www.adafruit.com/products/1603
data-sheet: http://ae-bst.resource.bosch.com/media/products/dokumente/
bmp180/BST-BMP180-DS000-09.pdf

The MIT License (MIT)
Copyright (c) 2014 Sebastian Plamauer, oeplse@gmail.com
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
'''

from ustruct import unpack as unp
from machine import I2C, Pin
import math
import time

# BMP180 class
class BMP180():
    '''
    Module for the BMP180 pressure sensor.
    '''

    _bmp_addr = 119             # adress of BMP180 is hardcoded on the sensor

    # init
    def __init__(self, i2c_bus):

        # create i2c obect
        _bmp_addr = self._bmp_addr
        self._bmp_i2c = i2c_bus
        self._bmp_i2c.start()
        self.chip_id = self._bmp_i2c.readfrom_mem(_bmp_addr, 0xD0, 2)
        # read calibration data from EEPROM
        self._AC1 = unp('>h', self._bmp_i2c.readfrom_mem(_bmp_addr, 0xAA, 2))[0]
        self._AC2 = unp('>h', self._bmp_i2c.readfrom_mem(_bmp_addr, 0xAC, 2))[0]
        self._AC3 = unp('>h', self._bmp_i2c.readfrom_mem(_bmp_addr, 0xAE, 2))[0]
        self._AC4 = unp('>H', self._bmp_i2c.readfrom_mem(_bmp_addr, 0xB0, 2))[0]
        self._AC5 = unp('>H', self._bmp_i2c.readfrom_mem(_bmp_addr, 0xB2, 2))[0]
        self._AC6 = unp('>H', self._bmp_i2c.readfrom_mem(_bmp_addr, 0xB4, 2))[0]
        self._B1 = unp('>h', self._bmp_i2c.readfrom_mem(_bmp_addr, 0xB6, 2))[0]
        self._B2 = unp('>h', self._bmp_i2c.readfrom_mem(_bmp_addr, 0xB8, 2))[0]
        self._MB = unp('>h', self._bmp_i2c.readfrom_mem(_bmp_addr, 0xBA, 2))[0]
        self._MC = unp('>h', self._bmp_i2c.readfrom_mem(_bmp_addr, 0xBC, 2))[0]
        self._MD = unp('>h', self._bmp_i2c.readfrom_mem(_bmp_addr, 0xBE, 2))[0]

        # settings to be adjusted by user
        self.oversample_setting = 3
        self.baseline = 101325.0

        # output raw
        self.UT_raw = None
        self.B5_raw = None
        self.MSB_raw = None
        self.LSB_raw = None
        self.XLSB_raw = None
        self.gauge = self.makegauge() # Generator instance
        for _ in range(128):
            next(self.gauge)
            time.sleep_ms(1)

    def compvaldump(self):
        '''
        Returns a list of all compensation values
        '''
        return [self._AC1, self._AC2, self._AC3, self._AC4, self._AC5, self._AC6,
                self._B1, self._B2, self._MB, self._MC, self._MD, self.oversample_setting]

    # gauge raw
    def makegauge(self):
        '''
        Generator refreshing the raw measurments.
        '''
        delays = (5, 8, 14, 25)
        while True:
            self._bmp_i2c.writeto_mem(self._bmp_addr, 0xF4, bytearray([0x2E]))
            t_start = time.ticks_ms()
            while (time.ticks_ms() - t_start) <= 5: # 5mS delay
                yield None
            try:
                self.UT_raw = self._bmp_i2c.readfrom_mem(self._bmp_addr, 0xF6, 2)
            except:
                yield None
            self._bmp_i2c.writeto_mem(self._bmp_addr, 0xF4, bytearray([0x34+(self.oversample_setting << 6)]))
            t_pressure_ready = delays[self.oversample_setting]
            t_start = time.ticks_ms()
            while (time.ticks_ms() - t_start) <= t_pressure_ready:
                yield None
            try:
                self.MSB_raw = self._bmp_i2c.readfrom_mem(self._bmp_addr, 0xF6, 1)
                self.LSB_raw = self._bmp_i2c.readfrom_mem(self._bmp_addr, 0xF7, 1)
                self.XLSB_raw = self._bmp_i2c.readfrom_mem(self._bmp_addr, 0xF8, 1)
            except:
                yield None
            yield True

    def blocking_read(self):
        if next(self.gauge) is not None: # Discard old data
            pass
        while next(self.gauge) is None:
            pass

    @property
    def oversample_sett(self):
        return self.oversample_setting

    @oversample_sett.setter
    def oversample_sett(self, value):
        if value in range(4):
            self.oversample_setting = value
        else:
            print('oversample_sett can only be 0, 1, 2 or 3, using 3 instead')
            self.oversample_setting = 3

    @property
    def temperature(self):
        '''
        Temperature in degree C.
        '''
        next(self.gauge)
        try:
            UT = unp('>H', self.UT_raw)[0]
        except:
            return 0.0
        X1 = (UT-self._AC6)*self._AC5/2**15
        X2 = self._MC*2**11/(X1+self._MD)
        self.B5_raw = X1+X2
        return (((X1+X2)+8)/2**4)/10

    @property
    def pressure(self):
        '''
        Pressure in mbar.
        '''
        next(self.gauge)
        self.temperature  # Populate self.B5_raw
        try:
            MSB = unp('B', self.MSB_raw)[0]
            LSB = unp('B', self.LSB_raw)[0]
            XLSB = unp('B', self.XLSB_raw)[0]
        except:
            return 0.0
        UP = ((MSB << 16)+(LSB << 8)+XLSB) >> (8-self.oversample_setting)
        B6 = self.B5_raw-4000
        X1 = (self._B2*(B6**2/2**12))/2**11
        X2 = self._AC2*B6/2**11
        X3 = X1+X2
        B3 = ((int((self._AC1*4+X3)) << self.oversample_setting)+2)/4
        X1 = self._AC3*B6/2**13
        X2 = (self._B1*(B6**2/2**12))/2**16
        X3 = ((X1+X2)+2)/2**2
        B4 = abs(self._AC4)*(X3+32768)/2**15
        B7 = (abs(UP)-B3) * (50000 >> self.oversample_setting)
        if B7 < 0x80000000:
            pressure = (B7*2)/B4
        else:
            pressure = (B7/B4)*2
        X1 = (pressure/2**8)**2
        X1 = (X1*3038)/2**16
        X2 = (-7357*pressure)/2**16
        return pressure+(X1+X2+3791)/2**4

    @property
    def altitude(self):
        '''
        Altitude in m.
        '''
        try:
            p = -7990.0*math.log(self.pressure/self.baseline)
        except:
            p = 0.0
        return p
<|endoftext|>


<commit_before>import os

import pytest

from updot import links

# TODO:
# make a fixture (that inherits 'fs') that mocks the singleton accessor

# TODO: def test__tracked_link_exists_with_correct_target__ignores():
#    """Rebuilding an existing symlink should do nothing"""

# TODO: def test__untracked_link_exists_with_correct_target__tracks_and_ignores():
#    """Should take over an already-correct symlink"""
#    note: be sure to test addition to state db

# TODO: def test__tracked_link_exists_with_different_target__updates():
#    """A symlink we were tracking has changed in spec, so update the symlink"""
#    note: be sure to test addition to state db

# TODO: def test__untracked_link_exists_with_different_target__throws():
#    """Symlink already exists and is pointing somewhere unexpected"""

def test__link_not_exist_and_target_exists__shortens_creates_and_tracks(fs):
    """Basic behavior of creating new symlinks"""

    fs.create_file(os.path.expanduser('~/path/to/actual/file.txt'), contents='abc')

    links.ln('~/path/to/link', '~/path/to/actual/file.txt')

    # ensure link is shortened and not absolute, and works
    expanded_link = os.path.expanduser('~/path/to/link')
    assert os.readlink(expanded_link).replace('\\', '/') == '../actual/file.txt'
    assert open(expanded_link, 'r').read() == 'abc'

    # TODO: test addition to state db

# TODO: def test__link_parent_not_exist__auto_creates():
#    """Auto-create any parent folders required to create the link"""

def test__target_not_exist__ignores(fs):  # pylint: disable=unused-argument
    """Ignore symlinks referring to nonexistent target paths (will be very common across OS's)"""

    links.ln('~/link', '~/no-file.txt')
    assert not os.path.exists(os.path.expanduser('~/link'))

# TODO: def test__dup_target_and_link__throws(fs):
#    """Catch accidental duplication of symlinks"""

# TODO: def test__unspecified_symlinks_found_in_any_link_parent__warns():
#    """Detect when user has added symlinks manually and forgotten to update spec"""
    # FUTURE: offer to add to spec
    # FUTURE: offer to ignore future warnings about this (store in local state file)

# TODO: def test__windows_symlink_creation_failed_due_to_no_admin__fatals_with_help():
#    """Let user know they should enable Developer Mode or run as admin"""
    # should be a permissions problem that leads to this

if __name__ == "__main__":
    pytest.main(__file__)
<commit_msg>Cache home to be a little more readable (though keep ~ for all updot calls to ensure that code is working)<commit_after>import os

import pytest

from updot import links

HOME = os.path.expanduser('~').replace('\\', '/')

# TODO:
# make a fixture (that inherits 'fs') that mocks the singleton accessor

# TODO: def test__tracked_link_exists_with_correct_target__ignores():
#    """Rebuilding an existing symlink should do nothing"""

# TODO: def test__untracked_link_exists_with_correct_target__tracks_and_ignores():
#    """Should take over an already-correct symlink"""
#    note: be sure to test addition to state db

# TODO: def test__tracked_link_exists_with_different_target__updates():
#    """A symlink we were tracking has changed in spec, so update the symlink"""
#    note: be sure to test addition to state db

# TODO: def test__untracked_link_exists_with_different_target__throws():
#    """Symlink already exists and is pointing somewhere unexpected"""

def test__link_not_exist_and_target_exists__shortens_creates_and_tracks(fs):
    """Basic behavior of creating new symlinks"""

    fs.create_file(f'{HOME}/path/to/actual/file.txt', contents='abc')

    links.ln('~/path/to/link', '~/path/to/actual/file.txt')

    # ensure link is shortened and not absolute, and works
    assert os.readlink(f'{HOME}/path/to/link').replace('\\', '/') == '../actual/file.txt'
    assert open(f'{HOME}/path/to/link', 'r').read() == 'abc'

    # TODO: test addition to state db

# TODO: def test__link_parent_not_exist__auto_creates():
#    """Auto-create any parent folders required to create the link"""

def test__target_not_exist__ignores(fs):  # pylint: disable=unused-argument
    """Ignore symlinks referring to nonexistent target paths (will be very common across OS's)"""

    links.ln('~/link', '~/no-file.txt')

    assert not os.path.exists(f'{HOME}/link')

# TODO: def test__dup_target_and_link__throws(fs):
#    """Catch accidental duplication of symlinks"""

# TODO: def test__unspecified_symlinks_found_in_any_link_parent__warns():
#    """Detect when user has added symlinks manually and forgotten to update spec"""
    # FUTURE: offer to add to spec
    # FUTURE: offer to ignore future warnings about this (store in local state file)

# TODO: def test__windows_symlink_creation_failed_due_to_no_admin__fatals_with_help():
#    """Let user know they should enable Developer Mode or run as admin"""
    # should be a permissions problem that leads to this

if __name__ == "__main__":
    pytest.main(__file__)
<|endoftext|>