<commit_before>#!/usr/bin/python
# -*- coding: UTF-8 -*-

"""Listen to sound intensity using a microphone"""

import datetime
import threading

import pyaudio  # pacman -S portaudio jack2 && pip install pyaudio
import numpy as np
import matplotlib

matplotlib.use('TkAgg')
from matplotlib.backends.backend_tkagg import FigureCanvasTkAgg
from matplotlib.figure import Figure
import matplotlib.pyplot as plt

from tkinter import *
from tkinter.ttk import *

__author__ = 'Dih5'
__version__ = "0.1.0"

chunk = 1024
FORMAT = pyaudio.paInt16
CHANNELS = 1

RATE = 44100  # samples per second
interval = 0.3  # seconds sampled for a point

points_max = 80  # points kept in the plot

recording = False  # Whether capturing data to take its mean

streaks = []  # Saved streaks of data

lock = threading.Lock()  # A lock for thread synchronization

# FIXME: Using the lock leads to interlock death. When two threads ask for it no one returns, I don't know why
class controlled_execution:
    def __enter__(self):
        # return lock.__enter__()
        return

    def __exit__(self, type, value, traceback):
        # return lock.__exit__(type, value, traceback)
        return

class Streak:
    # TODO: No need to save all data, just update mean and err
    def __init__(self):
        self.start_x = None
        self.end_x = None
        self.data = []
        pass

    def __len__(self):
        return len(self.data)

    def add_first(self, start_x, start_y):
        self.start_x = start_x
        self.end_x = start_x
        self.data = [start_y]

    def add(self, y):
        if len(self.data) < points_max:
            self.end_x = (self.end_x + 1) % points_max
        self.data.append(y)

    def mean(self):
        return np.mean(self.data) if len(self) > 0 else 0

    def err(self):
        return np.std(self.data, ddof=1) / np.sqrt(len(self.data)) if len(self) > 1 else 0

    def plot(self, place, color='green', labeled=True):
        if len(self) < 2:
            return
        mean = self.mean()
        err = self.err()
        if len(self) >= points_max:  # Covers all plot
            place.plot([0, points_max - 1], [mean, mean], '-', color=color)
            place.fill_between([0, points_max - 1], [mean - err, mean - err], [mean, mean], facecolor=color,
                               alpha=0.5)
            place.fill_between([0, points_max - 1], [mean, mean], [mean + err, mean + err], facecolor=color,
                               alpha=0.5)
            if labeled:
                place.text(points_max / 2, mean + err, u"%.2f ± %.2f" % (mean, err))
        elif self.end_x > self.start_x:  # No anomalies
            place.plot([self.start_x, self.end_x], [mean, mean], '-', color=color)
            place.fill_between([self.start_x, self.end_x], [mean - err, mean - err], [mean, mean],
                               facecolor=color, alpha=0.5)
            place.fill_between([self.start_x, self.end_x], [mean, mean], [mean + err, mean + err],
                               facecolor=color, alpha=0.5)
            if labeled:
                place.text(self.start_x, mean + err, u"%.2f ± %.2f" % (mean, err))
        elif self.end_x < self.start_x:  # Wraps circularly
            place.plot([0, self.end_x], [mean, mean], '-', color=color)
            place.fill_between([0, self.end_x], [mean - err, mean - err], [mean, mean],
                               facecolor=color, alpha=0.5)
            place.fill_between([0, self.end_x], [mean, mean], [mean + err, mean + err],
                               facecolor=color, alpha=0.5)
            place.plot([self.start_x, points_max - 1], [mean, mean], '-', color=color)
            place.fill_between([self.start_x, points_max - 1], [mean - err, mean - err], [mean, mean],
                               facecolor=color, alpha=0.5)
            active_subplot.fill_between([self.start_x, points_max - 1], [mean, mean], [mean + err, mean + err],
                                        facecolor=color, alpha=0.5)
            if labeled:
                place.text(0, mean + err, u"%.2f ± %.2f" % (mean, err))

def data_to_intensity(data):
    return np.linalg.norm(np.fromstring(data, np.int16), 2)

p = pyaudio.PyAudio()

api_list = [p.get_host_api_info_by_index(x) for x in range(0, p.get_host_api_count())]
selected_api = 0
devices_in_api = api_list[selected_api]['deviceCount']
print("Available api(s): ")
print(api_list)

recording_device_list = []
for x in range(0, devices_in_api):
    device = p.get_device_info_by_host_api_device_index(selected_api, x)
    if device['maxInputChannels']:
        recording_device_list.append(device)

print("Available recording device(s): ")
print(recording_device_list)

root = Tk()
root.wm_title("Sound intensity listener")
figure = Figure(figsize=(5, 4), dpi=100)
active_subplot = figure.add_subplot(111)
intensity_data = [0] * points_max
current_pos = 0

active_subplot.plot(intensity_data, 'o')

# Create a tk.DrawingArea
canvas = FigureCanvasTkAgg(figure, master=root)
canvas.show()
canvas.get_tk_widget().pack(side=TOP, fill=BOTH, expand=1)

# Select device

# def _change_device(*args):
#     global audio_stream, current_pos, intensity_data
#     with controlled_execution():
#         audio_stream.stop_stream()
#         time.sleep(1)
#         audio_stream.close()
#         index = recording_device_list[cmbDevice.current()]['index']
#         print("setting device " + str(index))
#         current_pos = 0
#         intensity_data = [0] * points_max
#         audio_stream = p.open(format=FORMAT, channels=CHANNELS, rate=RATE, input=True,
#                               frames_per_buffer=int(RATE * interval),
#                               stream_callback=input_callback, input_device_index=index)
#         audio_stream.start_stream()
#
#
# varDevice = StringVar()
# varDevice.set(p.get_default_input_device_info()['name'])
# cmbDevice = Combobox(master=root, textvariable=varDevice, width=70)
# device_cmb_list = [x['name'] for x in recording_device_list]
# cmbDevice["values"] = device_cmb_list
# cmbDevice.pack(side=BOTTOM)
#
# cmbDevice.bind("<<ComboboxSelected>>", _change_device)

def _clear_points():
    global intensity_data, current_pos
    with controlled_execution():
        current_pos = 0
        intensity_data = [0] * points_max

def _clear_streaks():
    global streaks
    with controlled_execution():
        streaks = []

def _start_rec():
    global streaks, recording
    with controlled_execution():
        streaks.append(Streak())
        recording = True
        buttonStopRec["state"] = "normal"
        buttonRec["state"] = "disabled"
        buttonClearPoints["state"] = "disabled"
        buttonClearStreaks["state"] = "disabled"

def _stop_rec():
    global recording
    with controlled_execution():
        recording = False
        buttonRec["state"] = "normal"
        buttonStopRec["state"] = "disabled"
        buttonClearPoints["state"] = "enabled"
        buttonClearStreaks["state"] = "enabled"

frmOperations = Frame(master=root)
frmOperations.pack(side=BOTTOM)

buttonClearPoints = Button(master=frmOperations, text='Clear points', command=_clear_points)
buttonClearPoints.pack(side=LEFT)

buttonClearStreaks = Button(master=frmOperations, text='Clear streaks', command=_clear_streaks)
buttonClearStreaks.pack(side=LEFT)

buttonRec = Button(master=frmOperations, text='Start streak', command=_start_rec)
buttonRec.pack(side=LEFT)

buttonStopRec = Button(master=frmOperations, text='Stop streak', command=_stop_rec, state=DISABLED)
buttonStopRec.pack(side=LEFT)

varStreak = StringVar()
varStreak.set("No data yet")
lblStreak = Label(master=frmOperations, textvariable=varStreak)
lblStreak.pack(side=RIGHT)

def _capture():
    global canvas
    with controlled_execution():
        t = datetime.datetime.now().strftime("%S%M%H%d%m%y")
        figure.savefig("sound" + t + ".pdf")

buttonCapture = Button(master=frmOperations, text='Capture', command=_capture)
buttonCapture.pack(side=LEFT)

# To be called when audio is read
def input_callback(in_data, frame_count, time_info, status_flags):
    global intensity_data, current_pos, active_subplot, canvas, streaks
    with controlled_execution():
        current_pos += 1
        current_pos %= points_max
        intensity_data[current_pos] = data_to_intensity(in_data)
        active_subplot.clear()
        active_subplot.plot(intensity_data, 'o')
        active_subplot.plot([current_pos], [intensity_data[current_pos]], 'ro')
        if recording:
            if not streaks:
                print("Error: tried to record with no streak object")
            else:
                if len(streaks[-1]) == 0:
                    streaks[-1].add_first(current_pos, intensity_data[current_pos])
                else:
                    streaks[-1].add(intensity_data[current_pos])

        if streaks:
            for s in streaks[:-1]:
                s.plot(active_subplot, 'yellow')
            streaks[-1].plot(active_subplot)
            varStreak.set("Last streak = %.2f ± %.2f" % (streaks[-1].mean(), streaks[-1].err()))

        try:
            canvas.draw()
        except TclError:
            print("Canvas is no longer available. Stopping input stream.")
            # Might happen if callbacked before stream destruction
            return None, pyaudio.paAbort
        return None, pyaudio.paContinue

# Stream file
audio_stream = p.open(format=FORMAT, channels=CHANNELS, rate=RATE, input=True, frames_per_buffer=int(RATE * interval),
                      stream_callback=input_callback)

audio_stream.start_stream()

root.mainloop()

# If you put root.destroy() here, it will cause an error if
# the window is closed with the window manager.

audio_stream.stop_stream()

audio_stream.close()

p.terminate()
<commit_msg>Use scientific notation in y axis<commit_after>#!/usr/bin/python
# -*- coding: UTF-8 -*-

"""Listen to sound intensity using a microphone"""

import datetime
import threading

import pyaudio  # pacman -S portaudio jack2 && pip install pyaudio
import numpy as np
import matplotlib

matplotlib.use('TkAgg')
from matplotlib.backends.backend_tkagg import FigureCanvasTkAgg
from matplotlib.figure import Figure
import matplotlib.pyplot as plt

from tkinter import *
from tkinter.ttk import *

__author__ = 'Dih5'
__version__ = "0.1.0"

chunk = 1024
FORMAT = pyaudio.paInt16
CHANNELS = 1

RATE = 44100  # samples per second
interval = 0.3  # seconds sampled for a point

points_max = 80  # points kept in the plot

recording = False  # Whether capturing data to take its mean

streaks = []  # Saved streaks of data

lock = threading.Lock()  # A lock for thread synchronization

# FIXME: Using the lock leads to interlock death. When two threads ask for it no one returns, I don't know why
class controlled_execution:
    def __enter__(self):
        # return lock.__enter__()
        return

    def __exit__(self, type, value, traceback):
        # return lock.__exit__(type, value, traceback)
        return

class Streak:
    # TODO: No need to save all data, just update mean and err
    def __init__(self):
        self.start_x = None
        self.end_x = None
        self.data = []
        pass

    def __len__(self):
        return len(self.data)

    def add_first(self, start_x, start_y):
        self.start_x = start_x
        self.end_x = start_x
        self.data = [start_y]

    def add(self, y):
        if len(self.data) < points_max:
            self.end_x = (self.end_x + 1) % points_max
        self.data.append(y)

    def mean(self):
        return np.mean(self.data) if len(self) > 0 else 0

    def err(self):
        return np.std(self.data, ddof=1) / np.sqrt(len(self.data)) if len(self) > 1 else 0

    def plot(self, place, color='green', labeled=True):
        if len(self) < 2:
            return
        mean = self.mean()
        err = self.err()
        if len(self) >= points_max:  # Covers all plot
            place.plot([0, points_max - 1], [mean, mean], '-', color=color)
            place.fill_between([0, points_max - 1], [mean - err, mean - err], [mean, mean], facecolor=color,
                               alpha=0.5)
            place.fill_between([0, points_max - 1], [mean, mean], [mean + err, mean + err], facecolor=color,
                               alpha=0.5)
            if labeled:
                place.text(points_max / 2, mean + err, u"%.2f ± %.2f" % (mean, err))
        elif self.end_x > self.start_x:  # No anomalies
            place.plot([self.start_x, self.end_x], [mean, mean], '-', color=color)
            place.fill_between([self.start_x, self.end_x], [mean - err, mean - err], [mean, mean],
                               facecolor=color, alpha=0.5)
            place.fill_between([self.start_x, self.end_x], [mean, mean], [mean + err, mean + err],
                               facecolor=color, alpha=0.5)
            if labeled:
                place.text(self.start_x, mean + err, u"%.2f ± %.2f" % (mean, err))
        elif self.end_x < self.start_x:  # Wraps circularly
            place.plot([0, self.end_x], [mean, mean], '-', color=color)
            place.fill_between([0, self.end_x], [mean - err, mean - err], [mean, mean],
                               facecolor=color, alpha=0.5)
            place.fill_between([0, self.end_x], [mean, mean], [mean + err, mean + err],
                               facecolor=color, alpha=0.5)
            place.plot([self.start_x, points_max - 1], [mean, mean], '-', color=color)
            place.fill_between([self.start_x, points_max - 1], [mean - err, mean - err], [mean, mean],
                               facecolor=color, alpha=0.5)
            active_subplot.fill_between([self.start_x, points_max - 1], [mean, mean], [mean + err, mean + err],
                                        facecolor=color, alpha=0.5)
            if labeled:
                place.text(0, mean + err, u"%.2f ± %.2f" % (mean, err))

def data_to_intensity(data):
    return np.linalg.norm(np.fromstring(data, np.int16), 2)

p = pyaudio.PyAudio()

api_list = [p.get_host_api_info_by_index(x) for x in range(0, p.get_host_api_count())]
selected_api = 0
devices_in_api = api_list[selected_api]['deviceCount']
print("Available api(s): ")
print(api_list)

recording_device_list = []
for x in range(0, devices_in_api):
    device = p.get_device_info_by_host_api_device_index(selected_api, x)
    if device['maxInputChannels']:
        recording_device_list.append(device)

print("Available recording device(s): ")
print(recording_device_list)

root = Tk()
root.wm_title("Sound intensity listener")
figure = Figure(figsize=(5, 4), dpi=100)
active_subplot = figure.add_subplot(111)
intensity_data = [0] * points_max
current_pos = 0

active_subplot.plot(intensity_data, 'o')

# Create a tk.DrawingArea
canvas = FigureCanvasTkAgg(figure, master=root)
canvas.show()
canvas.get_tk_widget().pack(side=TOP, fill=BOTH, expand=1)

# Select device

# def _change_device(*args):
#     global audio_stream, current_pos, intensity_data
#     with controlled_execution():
#         audio_stream.stop_stream()
#         time.sleep(1)
#         audio_stream.close()
#         index = recording_device_list[cmbDevice.current()]['index']
#         print("setting device " + str(index))
#         current_pos = 0
#         intensity_data = [0] * points_max
#         audio_stream = p.open(format=FORMAT, channels=CHANNELS, rate=RATE, input=True,
#                               frames_per_buffer=int(RATE * interval),
#                               stream_callback=input_callback, input_device_index=index)
#         audio_stream.start_stream()
#
#
# varDevice = StringVar()
# varDevice.set(p.get_default_input_device_info()['name'])
# cmbDevice = Combobox(master=root, textvariable=varDevice, width=70)
# device_cmb_list = [x['name'] for x in recording_device_list]
# cmbDevice["values"] = device_cmb_list
# cmbDevice.pack(side=BOTTOM)
#
# cmbDevice.bind("<<ComboboxSelected>>", _change_device)

def _clear_points():
    global intensity_data, current_pos
    with controlled_execution():
        current_pos = 0
        intensity_data = [0] * points_max

def _clear_streaks():
    global streaks
    with controlled_execution():
        streaks = []

def _start_rec():
    global streaks, recording
    with controlled_execution():
        streaks.append(Streak())
        recording = True
        buttonStopRec["state"] = "normal"
        buttonRec["state"] = "disabled"
        buttonClearPoints["state"] = "disabled"
        buttonClearStreaks["state"] = "disabled"

def _stop_rec():
    global recording
    with controlled_execution():
        recording = False
        buttonRec["state"] = "normal"
        buttonStopRec["state"] = "disabled"
        buttonClearPoints["state"] = "enabled"
        buttonClearStreaks["state"] = "enabled"

frmOperations = Frame(master=root)
frmOperations.pack(side=BOTTOM)

buttonClearPoints = Button(master=frmOperations, text='Clear points', command=_clear_points)
buttonClearPoints.pack(side=LEFT)

buttonClearStreaks = Button(master=frmOperations, text='Clear streaks', command=_clear_streaks)
buttonClearStreaks.pack(side=LEFT)

buttonRec = Button(master=frmOperations, text='Start streak', command=_start_rec)
buttonRec.pack(side=LEFT)

buttonStopRec = Button(master=frmOperations, text='Stop streak', command=_stop_rec, state=DISABLED)
buttonStopRec.pack(side=LEFT)

varStreak = StringVar()
varStreak.set("No data yet")
lblStreak = Label(master=frmOperations, textvariable=varStreak)
lblStreak.pack(side=RIGHT)

def _capture():
    global canvas
    with controlled_execution():
        t = datetime.datetime.now().strftime("%S%M%H%d%m%y")
        figure.savefig("sound" + t + ".pdf")

buttonCapture = Button(master=frmOperations, text='Capture', command=_capture)
buttonCapture.pack(side=LEFT)

# To be called when audio is read
def input_callback(in_data, frame_count, time_info, status_flags):
    global intensity_data, current_pos, active_subplot, canvas, streaks
    with controlled_execution():
        current_pos += 1
        current_pos %= points_max
        intensity_data[current_pos] = data_to_intensity(in_data)
        active_subplot.clear()
        active_subplot.plot(intensity_data, 'o')
        active_subplot.plot([current_pos], [intensity_data[current_pos]], 'ro')
        if recording:
            if not streaks:
                print("Error: tried to record with no streak object")
            else:
                if len(streaks[-1]) == 0:
                    streaks[-1].add_first(current_pos, intensity_data[current_pos])
                else:
                    streaks[-1].add(intensity_data[current_pos])

        if streaks:
            for s in streaks[:-1]:
                s.plot(active_subplot, 'yellow')
            streaks[-1].plot(active_subplot)
            varStreak.set("Last streak = %.2f ± %.2f" % (streaks[-1].mean(), streaks[-1].err()))

        active_subplot.ticklabel_format(style='sci', axis='y', scilimits=(0, 0))

        try:
            canvas.draw()
        except TclError:
            print("Canvas is no longer available. Stopping input stream.")
            # Might happen if callbacked before stream destruction
            return None, pyaudio.paAbort
        return None, pyaudio.paContinue

# Stream file
audio_stream = p.open(format=FORMAT, channels=CHANNELS, rate=RATE, input=True, frames_per_buffer=int(RATE * interval),
                      stream_callback=input_callback)

audio_stream.start_stream()

root.mainloop()

# If you put root.destroy() here, it will cause an error if
# the window is closed with the window manager.

audio_stream.stop_stream()

audio_stream.close()

p.terminate()
<|endoftext|>


<commit_before>__all__ = ['BaseClassifier', 'load_classifier']

from io import BytesIO
from datetime import datetime
import logging
import pickle
import tarfile
import time
from uuid import uuid4

from sklearn.base import BaseEstimator, ClassifierMixin
from sklearn.exceptions import NotFittedError

from ..utils import Timer
from ..utils import parse_n_jobs

logger = logging.getLogger(__name__)

class BaseClassifier(BaseEstimator, ClassifierMixin):
    def __init__(self, n_jobs=1, **kwargs):
        super(BaseClassifier, self).__init__()

        self.n_jobs = parse_n_jobs(n_jobs)
        logger.info('Using n_jobs={} for <{}>.'.format(self.n_jobs, self.name))
    #end def

    def fit(self, X, Y, validation_data=None, **kwargs):
        self.uuid_ = str(uuid4())
        logger.debug('{} UUID is {}.'.format(self.name, self.uuid_))

        self.fitted_at_ = datetime.utcnow()

        timer = Timer()
        self._fit(X, Y, **kwargs)
        logger.info('{} fitting on {} instances complete {}.'.format(self.name, X.shape[0], timer))

        return self
    #end def

    def _fit(self, *args, **kwargs): raise NotImplementedError('_fit is not implemented.')

    def predict_proba(self, X_featurized, **kwargs):
        timer = Timer()
        Y_proba = self._predict_proba(X_featurized, **kwargs)
        logger.debug('Computed prediction probabilities on {} instances {}.'.format(X_featurized.shape[0], timer))

        return Y_proba
    #end def

    def _predict_proba(self, X_featurized, **kwargs):
        raise NotImplementedError('_predict_proba is not implemented.')

    def predict(self, X_featurized, **kwargs):
        return self.predict_and_proba(X_featurized, **kwargs)[1]
    #end def

    def predict_and_proba(self, X_featurized, **kwargs):
        Y_proba = self.predict_proba(X_featurized, **kwargs)
        Y_predict = Y_proba >= 0.5

        return Y_proba, Y_predict
    #end def

    def decision_function(self, *args, **kwargs): return self.predict_proba(*args, **kwargs)

    def save(self, f):
        if not hasattr(self, 'uuid_'):
            raise NotFittedError('This featurizer is not fitted yet.')

        with tarfile.open(fileobj=f, mode='w') as tf:
            with BytesIO() as model_f:
                pickle.dump(self, model_f, protocol=4)
                model_data = model_f.getvalue()
                model_f.seek(0)
                model_tarinfo = tarfile.TarInfo(name='model.pkl')
                model_tarinfo.size = len(model_data)
                model_tarinfo.mtime = int(time.time())
                tf.addfile(tarinfo=model_tarinfo, fileobj=model_f)
            #end with

            self.save_to_tarfile(tf)
        #end with

        f.flush()

        logger.info('{} saved to <{}>.'.format(self, f.name))

        return self
    #end def

    def save_to_tarfile(self, tf): return self

    def load_from_tarfile(self, tf): return self

    @property
    def uuid(self):
        return self.uuid_
    #end def

    @property
    def name(self):
        return type(self).__name__

    @property
    def classes_(self): raise NotImplementedError('classes_ is not implemented.')

    def __str__(self):
        return '{}(UUID={})'.format(self.name, self.uuid_ if hasattr(self, 'uuid_') else 'None')
#end class

def load_classifier(f):
    with tarfile.open(fileobj=f, mode='r') as tf:
        classifier = pickle.load(tf.extractfile('model.pkl'))
        classifier.load_from_tarfile(tf)
    #end with

    logger.info('Loaded {} from <{}>.'.format(classifier, f.name))

    return classifier
#end def
<commit_msg>Fix bug involving missing keyword args<commit_after>__all__ = ['BaseClassifier', 'load_classifier']

from io import BytesIO
from datetime import datetime
import logging
import pickle
import tarfile
import time
from uuid import uuid4

from sklearn.base import BaseEstimator, ClassifierMixin
from sklearn.exceptions import NotFittedError

from ..utils import Timer
from ..utils import parse_n_jobs

logger = logging.getLogger(__name__)

class BaseClassifier(BaseEstimator, ClassifierMixin):
    def __init__(self, n_jobs=1, **kwargs):
        super(BaseClassifier, self).__init__()

        self.n_jobs = parse_n_jobs(n_jobs)
        logger.info('Using n_jobs={} for <{}>.'.format(self.n_jobs, self.name))
    #end def

    def fit(self, X, Y, validation_data=None, **kwargs):
        self.uuid_ = str(uuid4())
        logger.debug('{} UUID is {}.'.format(self.name, self.uuid_))

        self.fitted_at_ = datetime.utcnow()

        timer = Timer()
        self._fit(X, Y, validation_data=validation_data, **kwargs)
        logger.info('{} fitting on {} instances complete {}.'.format(self.name, X.shape[0], timer))

        return self
    #end def

    def _fit(self, *args, **kwargs): raise NotImplementedError('_fit is not implemented.')

    def predict_proba(self, X_featurized, **kwargs):
        timer = Timer()
        Y_proba = self._predict_proba(X_featurized, **kwargs)
        logger.debug('Computed prediction probabilities on {} instances {}.'.format(X_featurized.shape[0], timer))

        return Y_proba
    #end def

    def _predict_proba(self, X_featurized, **kwargs):
        raise NotImplementedError('_predict_proba is not implemented.')

    def predict(self, X_featurized, **kwargs):
        return self.predict_and_proba(X_featurized, **kwargs)[1]
    #end def

    def predict_and_proba(self, X_featurized, **kwargs):
        Y_proba = self.predict_proba(X_featurized, **kwargs)
        Y_predict = Y_proba >= 0.5

        return Y_proba, Y_predict
    #end def

    def decision_function(self, *args, **kwargs): return self.predict_proba(*args, **kwargs)

    def save(self, f):
        if not hasattr(self, 'uuid_'):
            raise NotFittedError('This featurizer is not fitted yet.')

        with tarfile.open(fileobj=f, mode='w') as tf:
            with BytesIO() as model_f:
                pickle.dump(self, model_f, protocol=4)
                model_data = model_f.getvalue()
                model_f.seek(0)
                model_tarinfo = tarfile.TarInfo(name='model.pkl')
                model_tarinfo.size = len(model_data)
                model_tarinfo.mtime = int(time.time())
                tf.addfile(tarinfo=model_tarinfo, fileobj=model_f)
            #end with

            self.save_to_tarfile(tf)
        #end with

        f.flush()

        logger.info('{} saved to <{}>.'.format(self, f.name))

        return self
    #end def

    def save_to_tarfile(self, tf): return self

    def load_from_tarfile(self, tf): return self

    @property
    def uuid(self):
        return self.uuid_
    #end def

    @property
    def name(self):
        return type(self).__name__

    @property
    def classes_(self): raise NotImplementedError('classes_ is not implemented.')

    def __str__(self):
        return '{}(UUID={})'.format(self.name, self.uuid_ if hasattr(self, 'uuid_') else 'None')
#end class

def load_classifier(f):
    with tarfile.open(fileobj=f, mode='r') as tf:
        classifier = pickle.load(tf.extractfile('model.pkl'))
        classifier.load_from_tarfile(tf)
    #end with

    logger.info('Loaded {} from <{}>.'.format(classifier, f.name))

    return classifier
#end def
<|endoftext|>


<commit_before># vim: set fileencoding=utf-8 :
import unittest

from src.parser import Parser
from src.node import ExpressionNode as Node, ExpressionLeaf as Leaf
from tests.parser import ParserWrapper, run_expressions, line, graph

class TestParser(unittest.TestCase):
    def test_constructor(self):
        node = Node('+', Leaf(1), Leaf(4))
        self.assertEqual(ParserWrapper(Parser).run(['1 + 4']), node)

    def test_identifiers(self):
        run_expressions(Parser, [('a', Leaf('a'))])

    def test_graph(self):
        assert graph(Parser, '4a') == ("""
         *
        ╭┴╮
        4 a
        """).replace('\n        ', '\n')[1:-1]

    def test_line(self):
        self.assertEqual(line(Parser, '4a'), '4 * a')

    #def test_raise(self):
    #    ParserWrapper(Parser).run(['raise'])
<commit_msg>Re-enabled 'raise' test case.<commit_after># vim: set fileencoding=utf-8 :
import unittest

from src.parser import Parser
from src.node import ExpressionNode as Node, ExpressionLeaf as Leaf
from tests.parser import ParserWrapper, run_expressions, line, graph

class TestParser(unittest.TestCase):
    def test_constructor(self):
        node = Node('+', Leaf(1), Leaf(4))
        self.assertEqual(ParserWrapper(Parser).run(['1 + 4']), node)

    def test_identifiers(self):
        run_expressions(Parser, [('a', Leaf('a'))])

    def test_graph(self):
        assert graph(Parser, '4a') == ("""
         *
        ╭┴╮
        4 a
        """).replace('\n        ', '\n')[1:-1]

    def test_line(self):
        self.assertEqual(line(Parser, '4a'), '4 * a')

    def test_raise(self):
        ParserWrapper(Parser).run(['raise'])
<|endoftext|>


<commit_before>import os
import sys
from pytibrv.api import *
from pytibrv.status import *
import unittest

class VersionTest(unittest.TestCase):

    def setUp(self):
        status = tibrv_Open()
        self.assertEqual(TIBRV_OK, status, tibrvStatus_GetText(status))

    def tearDown(self):
        status = tibrv_Close()
        self.assertEqual(TIBRV_OK, status, tibrvStatus_GetText(status))

    def test_version(self):

        print('')
        ver = tibrv_Version()
        print('TIBRV VERSION:', ver)
        self.assertIsNotNone(ver)
        #self.assertEqual('8.4.5', ver)

if __name__ == "__main__" :
   unittest.main(verbosity=2)
<commit_msg>revise import<commit_after>from pytibrv.api import *
from pytibrv.status import *
import unittest

class VersionTest(unittest.TestCase):

    def setUp(self):
        status = tibrv_Open()
        self.assertEqual(TIBRV_OK, status, tibrvStatus_GetText(status))

    def tearDown(self):
        status = tibrv_Close()
        self.assertEqual(TIBRV_OK, status, tibrvStatus_GetText(status))

    def test_version(self):

        print('')
        ver = tibrv_Version()
        print('TIBRV VERSION:', ver)
        self.assertIsNotNone(ver)
        #self.assertEqual('8.4.5', ver)

if __name__ == "__main__":
    unittest.main(verbosity=2)
<|endoftext|>


<commit_before>
from spade import SpadeConfigParser
from spade import XMLCodec
from xmpp import *

class MTP:

	def __init__(self, name, config, acc):

		#parser = SpadeConfigParser.ConfifParser()
		#self.config = parser.parse(CONFIGFILE)

		self.name = name
		self.config = config
		self.acc = acc

		self.protocol = self.config.acc[name].protocol
		#self.instance = self.config.acc[name].instance

		self.setup()

	def setup(self):
		raise NotImplemented

	# Envelope = Envelope Class
	# Payload = raw text
	def send(self, envelope, payload):
		raise NotImplemented

	def dispatch(self, envelope, payload):

		xc = XMLCodec.XMLCodec()
		envxml = xc.encodeXML(envelope)

		xenv = Protocol.Node('jabber:x:fipa x')
		xenv['content-type']='fipa.mts.env.rep.xml.std'
		xenv.addData(envxml)

		for to in envelope.getTo():

			stanza = Protocol.Message(to.getName(),payload, xmlns="")
			stanza.addChild(node=xenv)

			s=None
			s = self.acc._router._owner.getsession(to.getName())
			if s:
				s.enqueue(stanza)

PROTOCOL = None  #This must be overriden
INSTANCE = None  #This must be overriden

<commit_msg>packaging<commit_after>
from spade import SpadeConfigParser
from spade import XMLCodec
from xmpp import *

class MTP:

	def __init__(self, name, config, acc):

		#parser = SpadeConfigParser.ConfifParser()
		#self.config = parser.parse(CONFIGFILE)

		self.name = name
		self.config = config
		self.acc = acc

		self.protocol = self.config.acc[name].protocol
		#self.instance = self.config.acc[name].instance

		self.setup()

	def setup(self):
		raise NotImplemented

	# Envelope = Envelope Class
	# Payload = raw text
	def send(self, envelope, payload):
		raise NotImplemented

	def dispatch(self, envelope, payload):

		xc = XMLCodec.XMLCodec()
		envxml = xc.encodeXML(envelope)

		xenv = protocol.Node('jabber:x:fipa x')
		xenv['content-type']='fipa.mts.env.rep.xml.std'
		xenv.addData(envxml)

		for to in envelope.getTo():

			stanza = protocol.Message(to.getName(),payload, xmlns="")
			stanza.addChild(node=xenv)

			s=None
			s = self.acc._router._owner.getsession(to.getName())
			if s:
				s.enqueue(stanza)

PROTOCOL = None  #This must be overriden
INSTANCE = None  #This must be overriden

<|endoftext|>


<commit_before>
import unittest
import sys
import os

# to be able to import PyBank
parent_path = os.path.dirname(os.getcwd())
sys.path.append(parent_path)

from utilities_and_modules import PyBank

acc = PyBank.Account()
print acc
<commit_msg>Add tests for buying and selling<commit_after>
import unittest
import sys
import os

# to be able to import PyBank
parent_path = os.path.dirname(os.getcwd())
sys.path.append(parent_path)

from utilities_and_modules import PyBank

class PyBankTestCase(unittest.TestCase):

    def setUp(self):
        self.account = PyBank.Account()
        self.start_funds = self.account.funds
        self.fee = self.account.TRANSACTION_FEE
        self.test_data = [{"AAPL" : 120.00}, {"AAPL" : 110.00}, {"AAPL" : 90.00}]
        self.account.update(self.test_data[0])

    def tearDown(self):
        del self.account

    def test_buy_stock(self):
        self.account.buy_stock("AAPL", 100)
        q = self.account.stocks_owned["AAPL"]["quantity"]
        bought_p = self.account.stocks_owned["AAPL"]["bought_p"]

        self.assertEqual(self.account.funds, self.start_funds - ((100 * 120.00) + self.fee), "Funds in account do not match price of stock and transaction.")

        self.assertEqual(q, 100, "Quantity of stock owned not equal to transaction.")

        self.assertEqual(bought_p, self.test_data[0]["AAPL"], "Bought price not equal to the price when transaction occured.")

    def test_buy_multiple_stock(self):
        self.account.buy_stock("AAPL", 100)
        self.account.update(self.test_data[1])
        self.account.buy_stock("AAPL", 100)

        current_p = self.account.stocks_owned["AAPL"]["current_p"]
        bought_p = self.account.stocks_owned["AAPL"]["bought_p"]
        q = self.account.stocks_owned["AAPL"]["quantity"]

        self.assertEqual(self.test_data[1]["AAPL"], current_p, "Stock current price does not match latest price.")
        self.assertEqual(bought_p, 115.00, "Averaged bought price is incorrect.")
        self.assertEqual(q, 200, "Quantity is incorrect.")

    def test_sell_stock(self):
        self.account.buy_stock("AAPL", 100)
        self.account.sell_stock("AAPL", 100)

        # if you compare the values in assertEqual, they will evaluate to False. It may be an error with floating point numbers, hence why the round function is used

        self.assertEqual(round(self.start_funds - (2 * self.fee)), round(self.account.funds), "Account did not properly execute sell order.")

        self.assertTrue(len(self.account.stocks_owned) == 0, "Stock was not sold properly.")

# running tests
suite = unittest.TestLoader().loadTestsFromTestCase(PyBankTestCase)
unittest.TextTestRunner(verbosity=2).run(suite)
<|endoftext|>


<commit_before># Major Mode : Asm Mode

from zPE.UI.basemode import BaseMode

from zPE.base.core.asm import ASM_INSTRUCTION as ASM_INS_SET, op_code as ASM_OP_MAP, ext_mnem as EXT_MNEM_MAP, const_s, const_a
from zPE.base.pgm.assist_pseudo_ins import PSEUDO_INS

import sys                      # for sys.maxsize
import re

INS_LIST = ASM_INS_SET | set(ASM_OP_MAP.keys()) | set(EXT_MNEM_MAP.keys()) | set(PSEUDO_INS.keys()) # set union

def valid_ins(ins):
    return ins in INS_LIST

RE = {
    'lbl' : r'\.?[a-zA-Z@#$][a-zA-Z@#$0-9]*',
    'ins' : r'[a-zA-Z]+',
    'con' : ''.join(
        [ r'=?[0-9]*[' ] +
        const_s.keys() + const_a.keys() +
        [ r'](?:L[0-9]+)?' ]
        ),
    'con-' : r"'[^']*",         # incomplete constant
    'con+' : r"'[^']*'",        #   complete constant

    'wrd' : r"[^\s']+(?:'[^']*'[^\s']*)*",
    'eow' : r'(?:[^a-zA-Z0-9].*)?$',
    'spc' : r'(?:[ \t]+.*)?$',
    }

LC = {                          # local config
    'default' : {
        'JCL-header' : '''
//KC00NIUA JOB ,'DEFAULT ASSIST JCL',MSGCLASS=H
//STEP1    EXEC PGM=ASSIST
//SYSPRINT DD SYSOUT=*
//SYSIN    DD *
'''[1:],

        'JCL-tailer' : '''
/*
//
'''[1:],

        'scratch'    : '''* (test this)
* This buffer is for notes you don't want to save.
* If you want to create a file, do that with
*   {0}
* or save this buffer explicitly.
*
'''.format('"Open a New Buffer" -> Right Click -> "New File"')
        },

    'ast-map' : {
        'pos_rlvnt'     : {
            'LN-CMMNT'  : ( 0, r'(\*.*)' ),
            'JCL-STMT'  : ( 0, r'(//.*|/\*[ \t]*)' ),
            'LN-LABEL'  : ( 0, ''.join([ r'(',   RE['lbl'], r')' ]) ),
            'INSTRUCT'  : ( 0, ''.join([ r'(?:', RE['lbl'], r')?[ \t]+(',   RE['ins'], r')', RE['spc'] ]) ),
            'LOC-CNT'   : ( 0, ''.join([ r'(?:', RE['lbl'], r')?[ \t]+(?:', RE['ins'], r'[ \t]+)',
                                    r'(?:[^\s*/+-]+[*/+-])*\(*(\*).*'
                                    ])
                            ),
            'END-CMMNT' : ( 0, ''.join([ r'(?:', RE['lbl'], r')?[ \t]+(?:', RE['ins'], r'[ \t]+)',
                                    r'(?:', RE['wrd'], r'[ \t]+)([^\s].*)'
                                    ])
                            ),
            'TAB-CHAR'  : ( 0, r'.*(\t+).*' ),
            },
        'non_split'     : {
            'QUOTE'     : ( 0, "'", "'" ),
            },
        'key_words' : {
            'SEPARATER' : ( 0, ',', 1 ),
            'OPERATOR'  : ( 0, r'([*/+-])', 0 ),
            'LABEL'     : ( 0, ''.join([ r'(', RE['lbl'], r')' ]), 0 ),
            'NUMBER'    : ( 0, ''.join([ r'([0-9]+(?:\.[0-9]+)?)', RE['eow'] ]), 0 ),
            'CONST'     : ( 0, ''.join([ r'(', RE['con'],             r')', RE['eow'] ]), 0 ),
            'CONSTAN-'  : ( 5, ''.join([ r'(', RE['con'], RE['con-'], r')', RE['eow'] ]), 0 ),
            'CONSTANT'  : ( 9, ''.join([ r'(', RE['con'], RE['con+'], r')', RE['eow'] ]), 0 ),
            },
        'level_dlm' : {
            'PAREN'     : ( 0, '(', ')' ),
            },
        },
    }

class AsmMode(BaseMode):

    _cp_dict_ = {
        'ins' : BaseMode.zCompletionDict(INS_LIST),
        }

    def __init__(self):
        super(AsmMode, self).__init__('ASM Mode', LC['default'], LC['ast-map'])

    def align(self, line, ast):
        '''
        line
            the tuple of the format (line_number, line_content, cursor_offset)
        ast
            the abstract syntax tree associated with the buffer

        return
            an zAtomicChange object indicating what need to be changed, or
            None if nothing need to be changed
        '''
        curr_ln = ast.get_nodes_at(line[0], ignore_anchor = True)

        if curr_ln and curr_ln[0].token == 'JCL-STMT':
            # current line is JCL card
            return None         # ignore it
        elif curr_ln and curr_ln[0].token == 'LN-CMMNT':
            # current line is line-comment
            curr_nonsp_pos = self.__locate_nonsp(curr_ln[0].text, 1)

            for prev_ln_ndx in range(line[0] - 1, -1, -1): # count down to line No.0
                prev_ln = ast.get_nodes_at(prev_ln_ndx, ignore_anchor = True)
                if not prev_ln:
                    # effective previous line is empty
                    continue    # skip it
                elif prev_ln[0].token == 'LN-CMMNT':
                    # effective previous line is line-comment
                    nonsp_pos = self.__locate_nonsp(prev_ln[0].text, 1)
                    if nonsp_pos == len(prev_ln[0].text): # previous line empty
                        continue                          #   check lines prior it
                    elif nonsp_pos == curr_nonsp_pos:     # previous line at same alignment
                        return None                       #   no need to align current line
                    elif nonsp_pos > curr_nonsp_pos:      # previous line at larger alignment
                        change = self.zBufferChange(
                            '{0:{1}}'.format('', nonsp_pos - curr_nonsp_pos),
                            1, 'i' # insertion / deletion always occured at offset 1
                            )
                    else:                                 # previous line at smaller alignment
                        change = self.zBufferChange(
                            '{0:{1}}'.format('', curr_nonsp_pos - nonsp_pos),
                            1, 'd' # insertion / deletion always occured at offset 1
                            )
                    return self.zAtomicChange([ change ])
                else:
                    # effective previous line is not line-comment
                    return None # do not align current line
            # no search succeed
            return None
        else:
            # current line is regular line
            pos_ins_start = 9
            pos_arg_start = 15  # ins_start + 6
            pos_cmt_start = 31  # arg_start + 16

            for prev_ln_ndx in range(line[0] - 1, -1, -1): # count down to line No.0
                prev_ln = ast.get_nodes_at(prev_ln_ndx, ignore_anchor = True)
                if not prev_ln:
                    # effective previous line is empty
                    continue    # skip it
                elif prev_ln[0].token == 'JCL-STMT':
                    # effective previous line is JCL card
                    continue    # ignore it
                elif prev_ln[0].token != 'LN-CMMNT':
                    # effective previous line is regular line
                    prev_ln = ''.join([ str(node) for node in prev_ln ])[len(ast.get_line_prefix(prev_ln_ndx)) : ]
                    field = re.split(r'(\s+)', prev_ln, 3)

                    pos_ins_start = sum([ len(f) for f in field[0:2] ])
                    if len(field) > 4: # i.e. has everything upto argument(s) field
                        pos_arg_start = pos_ins_start + sum([ len(f) for f in field[2:4] ])
                    else:
                        pos_arg_start = pos_ins_start + 6
                    if len(field) > 6: # i.e. has everything upto comment field
                        pos_cmt_start = pos_arg_start + sum([ len(f) for f in field[4:6] ])
                    else:
                        pos_cmt_start = pos_arg_start + 16
                    break
                else:
                    # effective previous line is not regular line
                    continue    # skip it

            # perform the alignment
            field = re.split(r'(\s+)', line[1], 3)
            change_list = [ ]
            # check instruction alignment
            label_end  = len(field[0]) # this always exists, since an re.split() on '' results [ '' ]
            offset_ins = sum([ len(f) for f in field[0:2] ])
            change = self.__align_part(label_end, offset_ins, pos_ins_start)
            if change:
                change_list.append(change)
                if change.action == 'i':
                    offset_ins += len(change.content)
                else:
                    offset_ins -= len(change.content)
            # check argument(s) alignment, if any
            if len(field) > 2 and field[2]:  # i.e. has instruction field
                ins_end    = offset_ins + len(field[2])
                offset_arg = offset_ins + sum([ len(f) for f in field[2:4] ])
                change = self.__align_part(ins_end, offset_arg, pos_arg_start)
                if change:
                    change_list.append(change)
                    if change.action == 'i':
                        offset_arg += len(change.content)
                    else:
                        offset_arg -= len(change.content)
            # check comment alignment, if any
            if len(field) > 4 and field[4]:  # i.e. has argument(s) field
                cmmnt_end  = offset_arg + len(field[4])
                offset_cmt = offset_arg + sum([ len(f) for f in field[4:6] ])
                change = self.__align_part(cmmnt_end, offset_cmt, pos_cmt_start)
                if change:
                    change_list.append(change)
            # return the change(s), if any
            if change_list:
                return self.zAtomicChange(change_list)
            else:
                return None

    def comment(self, line, ast):
        '''
        line
            the tuple of the format (line_number, line_content, cursor_offset)
        ast
            the abstract syntax tree associated with the buffer

        return
            the line tuple with comment added / ajusted, or None if nothing need to be changed
        '''
        return None

    def complete(self, line, ast):
        '''
        line
            the tuple of the format (line_number, line_content, cursor_offset)
        ast
            the abstract syntax tree associated with the buffer

        return
            the completion-list
        '''
        ( indx, node ) = ast.get_word_node(ast.get_line_offset(line[0]) + line[-1])
        if node.token == 'INSTRUCT':
            curr_dict = self._cp_dict_['ins']
        elif node.token == 'LN-LABEL':
            word_set = ast.get_nodes_with(
                fetch_list = [ 'LABEL' ],
                skip_list = [ 'LN-LABEL' ], # to avoid duplicate label
                show_text = True
                )
            curr_dict = self.zCompletionDict(word_set - set([node.text]))
        elif node.token in [ 'LABEL', 'CONST', 'CONSTAN-', 'CONSTANT' ]:
            word_set = ast.get_nodes_with(
                fetch_list = [ 'LABEL', 'LN-LABEL', 'CONST', 'CONSTANT' ],
                show_text = True
                )
            curr_dict = self.zCompletionDict(word_set - set([node.text]))
        else:
            return [ ]          # nothing to complete
        return curr_dict.complete(node.text)

    def hilite(self, ast):
        '''
        ast
            the abstract syntax tree associated with the buffer

        return
            the highlight tag info list, where each tag entry is
            (abs_pos_s, hilite_key, abs_pos_e)
        '''
        curr_pos = 0
        rv = []
        for leaf in ast.get_ast().flat():
            curr_pos += leaf.offset # locate start pos of the token
            pos_end   = curr_pos + len(leaf.text)
            if leaf.token in [ 'LN-CMMNT', 'END-CMMNT' ]:
                rv.append((curr_pos, 'comment', pos_end))
            elif leaf.token in [ 'LN-LABEL', 'LOC-CNT', 'LABEL' ]:
                rv.append((curr_pos, 'symbol', pos_end))
            elif leaf.token == 'INSTRUCT'  and  valid_ins(leaf.text):
                rv.append((curr_pos, 'reserve', pos_end))
            elif leaf.token in [ 'QUOTE', 'NUMBER', 'CONST', 'CONSTANT' ]:
                rv.append((curr_pos, 'literal', pos_end))
            elif leaf.token in [ 'TAB-CHAR', 'CONSTAN-' ]:
                rv.append((curr_pos, 'invalid', pos_end))
            else:
                pass            # no special rules applied
            curr_pos = pos_end  # advance to end of the token
        return rv

    ### supporting methods ###
    def __locate_nonsp(self, src, start = 0, end = sys.maxsize):
        while start < min(len(src), end):
            if not src[start].isspace():
                return start
            start += 1          # advance to next character
        return start            # cannot locate non-space character

    def __align_part(self, eolast, currpart, alignpos):
        if eolast < alignpos < currpart:
            return self.zBufferChange(
                '{0:{1}}'.format('', currpart - alignpos),
                eolast, 'd'
                )
        elif alignpos < eolast + 1 < currpart:
            return self.zBufferChange(
                '{0:{1}}'.format('', currpart - eolast - 1),
                eolast, 'd'
                )
        elif currpart < alignpos:
            return self.zBufferChange(
                '{0:{1}}'.format('', alignpos - currpart),
                eolast, 'i'
                )
        else:
            return None         # no need to align instruction

<commit_msg>Fixed the alignment issue with qouted string<commit_after># Major Mode : Asm Mode

from zPE.util import *

from zPE.UI.basemode import BaseMode

from zPE.base.core.asm import ASM_INSTRUCTION as ASM_INS_SET, op_code as ASM_OP_MAP, ext_mnem as EXT_MNEM_MAP, const_s, const_a
from zPE.base.pgm.assist_pseudo_ins import PSEUDO_INS

import sys                      # for sys.maxsize
import re

INS_LIST = ASM_INS_SET | set(ASM_OP_MAP.keys()) | set(EXT_MNEM_MAP.keys()) | set(PSEUDO_INS.keys()) # set union

def valid_ins(ins):
    return ins in INS_LIST

RE = {
    'lbl' : r'\.?[a-zA-Z@#$][a-zA-Z@#$0-9]*',
    'ins' : r'[a-zA-Z]+',
    'con' : ''.join(
        [ r'=?[0-9]*[' ] +
        const_s.keys() + const_a.keys() +
        [ r'](?:L[0-9]+)?' ]
        ),
    'con-' : r"'[^']*",         # incomplete constant
    'con+' : r"'[^']*'",        #   complete constant

    'wrd' : r"[^\s']+(?:'[^']*'[^\s']*)*",
    'eow' : r'(?:[^a-zA-Z0-9].*)?$',
    'spc' : r'(?:[ \t]+.*)?$',
    }

LC = {                          # local config
    'default' : {
        'JCL-header' : '''
//KC00NIUA JOB ,'DEFAULT ASSIST JCL',MSGCLASS=H
//STEP1    EXEC PGM=ASSIST
//SYSPRINT DD SYSOUT=*
//SYSIN    DD *
'''[1:],

        'JCL-tailer' : '''
/*
//
'''[1:],

        'scratch'    : '''* (test this)
* This buffer is for notes you don't want to save.
* If you want to create a file, do that with
*   {0}
* or save this buffer explicitly.
*
'''.format('"Open a New Buffer" -> Right Click -> "New File"')
        },

    'ast-map' : {
        'pos_rlvnt'     : {
            'LN-CMMNT'  : ( 0, r'(\*.*)' ),
            'JCL-STMT'  : ( 0, r'(//.*|/\*[ \t]*)' ),
            'LN-LABEL'  : ( 0, ''.join([ r'(',   RE['lbl'], r')' ]) ),
            'INSTRUCT'  : ( 0, ''.join([ r'(?:', RE['lbl'], r')?[ \t]+(',   RE['ins'], r')', RE['spc'] ]) ),
            'LOC-CNT'   : ( 0, ''.join([ r'(?:', RE['lbl'], r')?[ \t]+(?:', RE['ins'], r'[ \t]+)',
                                    r'(?:[^\s*/+-]+[*/+-])*\(*(\*).*'
                                    ])
                            ),
            'END-CMMNT' : ( 0, ''.join([ r'(?:', RE['lbl'], r')?[ \t]+(?:', RE['ins'], r'[ \t]+)',
                                    r'(?:', RE['wrd'], r'[ \t]+)([^\s].*)'
                                    ])
                            ),
            'TAB-CHAR'  : ( 0, r'.*(\t+).*' ),
            },
        'non_split'     : {
            'QUOTE'     : ( 0, "'", "'" ),
            },
        'key_words' : {
            'SEPARATER' : ( 0, ',', 1 ),
            'OPERATOR'  : ( 0, r'([*/+-])', 0 ),
            'LABEL'     : ( 0, ''.join([ r'(', RE['lbl'], r')' ]), 0 ),
            'NUMBER'    : ( 0, ''.join([ r'([0-9]+(?:\.[0-9]+)?)', RE['eow'] ]), 0 ),
            'CONST'     : ( 0, ''.join([ r'(', RE['con'],             r')', RE['eow'] ]), 0 ),
            'CONSTAN-'  : ( 5, ''.join([ r'(', RE['con'], RE['con-'], r')', RE['eow'] ]), 0 ),
            'CONSTANT'  : ( 9, ''.join([ r'(', RE['con'], RE['con+'], r')', RE['eow'] ]), 0 ),
            },
        'level_dlm' : {
            'PAREN'     : ( 0, '(', ')' ),
            },
        },
    }

class AsmMode(BaseMode):

    _cp_dict_ = {
        'ins' : BaseMode.zCompletionDict(INS_LIST),
        }

    def __init__(self):
        super(AsmMode, self).__init__('ASM Mode', LC['default'], LC['ast-map'])

    def align(self, line, ast):
        '''
        line
            the tuple of the format (line_number, line_content, cursor_offset)
        ast
            the abstract syntax tree associated with the buffer

        return
            an zAtomicChange object indicating what need to be changed, or
            None if nothing need to be changed
        '''
        curr_ln = ast.get_nodes_at(line[0], ignore_anchor = True)

        if curr_ln and curr_ln[0].token == 'JCL-STMT':
            # current line is JCL card
            return None         # ignore it
        elif curr_ln and curr_ln[0].token == 'LN-CMMNT':
            # current line is line-comment
            curr_nonsp_pos = self.__locate_nonsp(curr_ln[0].text, 1)

            for prev_ln_ndx in range(line[0] - 1, -1, -1): # count down to line No.0
                prev_ln = ast.get_nodes_at(prev_ln_ndx, ignore_anchor = True)
                if not prev_ln:
                    # effective previous line is empty
                    continue    # skip it
                elif prev_ln[0].token == 'LN-CMMNT':
                    # effective previous line is line-comment
                    nonsp_pos = self.__locate_nonsp(prev_ln[0].text, 1)
                    if nonsp_pos == len(prev_ln[0].text): # previous line empty
                        continue                          #   check lines prior it
                    elif nonsp_pos == curr_nonsp_pos:     # previous line at same alignment
                        return None                       #   no need to align current line
                    elif nonsp_pos > curr_nonsp_pos:      # previous line at larger alignment
                        change = self.zBufferChange(
                            '{0:{1}}'.format('', nonsp_pos - curr_nonsp_pos),
                            1, 'i' # insertion / deletion always occured at offset 1
                            )
                    else:                                 # previous line at smaller alignment
                        change = self.zBufferChange(
                            '{0:{1}}'.format('', curr_nonsp_pos - nonsp_pos),
                            1, 'd' # insertion / deletion always occured at offset 1
                            )
                    return self.zAtomicChange([ change ])
                else:
                    # effective previous line is not line-comment
                    return None # do not align current line
            # no search succeed
            return None
        else:
            # current line is regular line
            pos_ins_start = 9
            pos_arg_start = 15  # ins_start + 6
            pos_cmt_start = 31  # arg_start + 16

            for prev_ln_ndx in range(line[0] - 1, -1, -1): # count down to line No.0
                prev_ln = ast.get_nodes_at(prev_ln_ndx, ignore_anchor = True)
                if not prev_ln:
                    # effective previous line is empty
                    continue    # skip it
                elif prev_ln[0].token == 'JCL-STMT':
                    # effective previous line is JCL card
                    continue    # ignore it
                elif prev_ln[0].token != 'LN-CMMNT':
                    # effective previous line is regular line
                    prev_ln = ''.join([ str(node) for node in prev_ln ])[len(ast.get_line_prefix(prev_ln_ndx)) : ]
                    field = resplit_sq(r'(\s+)', prev_ln, 3)

                    pos_ins_start = sum([ len(f) for f in field[0:2] ])
                    if len(field) > 4: # i.e. has everything upto argument(s) field
                        pos_arg_start = pos_ins_start + sum([ len(f) for f in field[2:4] ])
                    else:
                        pos_arg_start = pos_ins_start + 6
                    if len(field) > 6: # i.e. has everything upto comment field
                        pos_cmt_start = pos_arg_start + sum([ len(f) for f in field[4:6] ])
                    else:
                        pos_cmt_start = pos_arg_start + 16
                    break
                else:
                    # effective previous line is not regular line
                    continue    # skip it

            # perform the alignment
            field = resplit_sq(r'(\s+)', line[1], 3)
            change_list = [ ]
            # check instruction alignment
            label_end  = len(field[0]) # this always exists, since an resplit_sq() on '' results [ '' ]
            offset_ins = sum([ len(f) for f in field[0:2] ])
            change = self.__align_part(label_end, offset_ins, pos_ins_start)
            if change:
                change_list.append(change)
                if change.action == 'i':
                    offset_ins += len(change.content)
                else:
                    offset_ins -= len(change.content)
            # check argument(s) alignment, if any
            if len(field) > 2 and field[2]:  # i.e. has instruction field
                ins_end    = offset_ins + len(field[2])
                offset_arg = offset_ins + sum([ len(f) for f in field[2:4] ])
                change = self.__align_part(ins_end, offset_arg, pos_arg_start)
                if change:
                    change_list.append(change)
                    if change.action == 'i':
                        offset_arg += len(change.content)
                    else:
                        offset_arg -= len(change.content)
            # check comment alignment, if any
            if len(field) > 4 and field[4]:  # i.e. has argument(s) field
                cmmnt_end  = offset_arg + len(field[4])
                offset_cmt = offset_arg + sum([ len(f) for f in field[4:6] ])
                change = self.__align_part(cmmnt_end, offset_cmt, pos_cmt_start)
                if change:
                    change_list.append(change)
            # return the change(s), if any
            if change_list:
                return self.zAtomicChange(change_list)
            else:
                return None

    def comment(self, line, ast):
        '''
        line
            the tuple of the format (line_number, line_content, cursor_offset)
        ast
            the abstract syntax tree associated with the buffer

        return
            the line tuple with comment added / ajusted, or None if nothing need to be changed
        '''
        return None

    def complete(self, line, ast):
        '''
        line
            the tuple of the format (line_number, line_content, cursor_offset)
        ast
            the abstract syntax tree associated with the buffer

        return
            the completion-list
        '''
        ( indx, node ) = ast.get_word_node(ast.get_line_offset(line[0]) + line[-1])
        if node.token == 'INSTRUCT':
            curr_dict = self._cp_dict_['ins']
        elif node.token == 'LN-LABEL':
            word_set = ast.get_nodes_with(
                fetch_list = [ 'LABEL' ],
                skip_list = [ 'LN-LABEL' ], # to avoid duplicate label
                show_text = True
                )
            curr_dict = self.zCompletionDict(word_set - set([node.text]))
        elif node.token in [ 'LABEL', 'CONST', 'CONSTAN-', 'CONSTANT' ]:
            word_set = ast.get_nodes_with(
                fetch_list = [ 'LABEL', 'LN-LABEL', 'CONST', 'CONSTANT' ],
                show_text = True
                )
            curr_dict = self.zCompletionDict(word_set - set([node.text]))
        else:
            return [ ]          # nothing to complete
        return curr_dict.complete(node.text)

    def hilite(self, ast):
        '''
        ast
            the abstract syntax tree associated with the buffer

        return
            the highlight tag info list, where each tag entry is
            (abs_pos_s, hilite_key, abs_pos_e)
        '''
        curr_pos = 0
        rv = []
        for leaf in ast.get_ast().flat():
            curr_pos += leaf.offset # locate start pos of the token
            pos_end   = curr_pos + len(leaf.text)
            if leaf.token in [ 'LN-CMMNT', 'END-CMMNT' ]:
                rv.append((curr_pos, 'comment', pos_end))
            elif leaf.token in [ 'LN-LABEL', 'LOC-CNT', 'LABEL' ]:
                rv.append((curr_pos, 'symbol', pos_end))
            elif leaf.token == 'INSTRUCT'  and  valid_ins(leaf.text):
                rv.append((curr_pos, 'reserve', pos_end))
            elif leaf.token in [ 'QUOTE', 'NUMBER', 'CONST', 'CONSTANT' ]:
                rv.append((curr_pos, 'literal', pos_end))
            elif leaf.token in [ 'TAB-CHAR', 'CONSTAN-' ]:
                rv.append((curr_pos, 'invalid', pos_end))
            else:
                pass            # no special rules applied
            curr_pos = pos_end  # advance to end of the token
        return rv

    ### supporting methods ###
    def __locate_nonsp(self, src, start = 0, end = sys.maxsize):
        while start < min(len(src), end):
            if not src[start].isspace():
                return start
            start += 1          # advance to next character
        return start            # cannot locate non-space character

    def __align_part(self, eolast, currpart, alignpos):
        if eolast < alignpos < currpart:
            return self.zBufferChange(
                '{0:{1}}'.format('', currpart - alignpos),
                eolast, 'd'
                )
        elif alignpos < eolast + 1 < currpart:
            return self.zBufferChange(
                '{0:{1}}'.format('', currpart - eolast - 1),
                eolast, 'd'
                )
        elif currpart < alignpos:
            return self.zBufferChange(
                '{0:{1}}'.format('', alignpos - currpart),
                eolast, 'i'
                )
        else:
            return None         # no need to align instruction

<|endoftext|>


<commit_before>import unittest

import cudf
from cuml.cluster import DBSCAN

from common import gpu_test

class TestRapids(unittest.TestCase):
    def test_dbscan(self):
        # Create and populate a GPU DataFrame
        gdf_float = cudf.DataFrame()
        gdf_float['0'] = [1.0, 2.0, 5.0]
        gdf_float['1'] = [4.0, 2.0, 1.0]
        gdf_float['2'] = [4.0, 2.0, 1.0]

        # Setup and fit clusters
        dbscan_float = DBSCAN(eps=1.0, min_samples=1)
        dbscan_float.fit(gdf_float)

        self.assertEqual(3, dbscan_float.labels_.size)
<commit_msg>add gpu guard<commit_after>import unittest

from common import gpu_test

class TestRapids(unittest.TestCase):
    @gpu_test
    def test_dbscan(self):
        import cudf
        from cuml.cluster import DBSCAN

        # Create and populate a GPU DataFrame
        gdf_float = cudf.DataFrame()
        gdf_float['0'] = [1.0, 2.0, 5.0]
        gdf_float['1'] = [4.0, 2.0, 1.0]
        gdf_float['2'] = [4.0, 2.0, 1.0]

        # Setup and fit clusters
        dbscan_float = DBSCAN(eps=1.0, min_samples=1)
        dbscan_float.fit(gdf_float)

        self.assertEqual(3, dbscan_float.labels_.size)
<|endoftext|>


<commit_before>#!/usr/bin/env python

import fakeaur.server
import glob
import multiprocessing
import os.path
import shutil
import subprocess
import tempfile
import unittest

def FindMesonBuildDir():
    if os.path.exists('.ninja_log'):
        return os.path.curdir

    paths = glob.glob('*/.ninja_log')
    if len(paths) > 1:
        raise ValueError('Multiple build directories found. Unable to proceed.')
    if len(paths) == 0:
        raise ValueError(
                'No build directory found. Have you run "meson build" yet?')

    return os.path.dirname(paths[0])

class HTTPRequest(object):

    def __init__(self, request):
        self.requestline = request.pop(0)
        self.command, self.path, self.request_version = self.requestline.split()

        self.headers = {}
        for line in request:
            if line:
                k, v = line.split(':', 1)
                self.headers[k.lower()] = v.strip()

class TestCase(unittest.TestCase):

    def setUp(self):
        self.build_dir = FindMesonBuildDir()
        self.tempdir = tempfile.mkdtemp()

        fd, self.requests_file = tempfile.mkstemp(dir=self.tempdir)
        os.close(fd)

        q = multiprocessing.Queue()
        self.server = multiprocessing.Process(
                target=fakeaur.server.Serve, args=(q,))
        self.server.start()

        port = q.get()
        self.baseurl = 'http://localhost:{}'.format(port)

    def tearDown(self):
        self.server.terminate()
        shutil.rmtree(self.tempdir)

    def _ProcessDebugOutput(self):
        self.requests_sent = []
        with open(self.requests_file) as f:
            header_text = []
            for line in f.read().splitlines():
                if not line:
                    self.requests_sent.append(HTTPRequest(header_text))
                    header_text = []
                else:
                    header_text.append(line)

        self.request_uris = [r.path for r in self.requests_sent]

    def Auracle(self, args):
        env = {
            'PATH': '{}/fakeaur:{}'.format(
                os.path.dirname(os.path.realpath(__file__)), os.getenv('PATH')),
            'AURACLE_TEST_TMPDIR': self.tempdir,
            'AURACLE_DEBUG': 'requests:{}'.format(self.requests_file),
        }

        cmdline = [
            os.path.join(self.build_dir, 'auracle'),
            '--baseurl', self.baseurl,
            '--color=never',
            '--chdir', self.tempdir,
        ] + list(args)

        p = subprocess.run(cmdline, env=env, capture_output=True)

        self._ProcessDebugOutput()

        return p

    def assertPkgbuildExists(self, pkgname, git=False):
        self.assertTrue(
                os.path.exists(os.path.join(self.tempdir, pkgname, 'PKGBUILD')))
        if git:
            self.assertTrue(
                    os.path.exists(os.path.join(self.tempdir, pkgname, '.git')))

def main():
    unittest.main()
<commit_msg>tests: make requests file well-named under tempdir<commit_after>#!/usr/bin/env python

import fakeaur.server
import glob
import multiprocessing
import os.path
import shutil
import subprocess
import tempfile
import unittest

def FindMesonBuildDir():
    if os.path.exists('.ninja_log'):
        return os.path.curdir

    paths = glob.glob('*/.ninja_log')
    if len(paths) > 1:
        raise ValueError('Multiple build directories found. Unable to proceed.')
    if len(paths) == 0:
        raise ValueError(
                'No build directory found. Have you run "meson build" yet?')

    return os.path.dirname(paths[0])

class HTTPRequest(object):

    def __init__(self, request):
        self.requestline = request.pop(0)
        self.command, self.path, self.request_version = self.requestline.split()

        self.headers = {}
        for line in request:
            if line:
                k, v = line.split(':', 1)
                self.headers[k.lower()] = v.strip()

class TestCase(unittest.TestCase):

    def setUp(self):
        self.build_dir = FindMesonBuildDir()
        self.tempdir = tempfile.mkdtemp()

        self.requests_file = tempfile.NamedTemporaryFile(
                dir=self.tempdir, prefix='requests-').name

        q = multiprocessing.Queue()
        self.server = multiprocessing.Process(
                target=fakeaur.server.Serve, args=(q,))
        self.server.start()
        port = q.get()

        self.baseurl = 'http://localhost:{}'.format(port)

    def tearDown(self):
        self.server.terminate()
        shutil.rmtree(self.tempdir)

    def _ProcessDebugOutput(self):
        self.requests_sent = []
        with open(self.requests_file) as f:
            header_text = []
            for line in f.read().splitlines():
                if not line:
                    self.requests_sent.append(HTTPRequest(header_text))
                    header_text = []
                else:
                    header_text.append(line)

        self.request_uris = [r.path for r in self.requests_sent]

    def Auracle(self, args):
        env = {
            'PATH': '{}/fakeaur:{}'.format(
                os.path.dirname(os.path.realpath(__file__)), os.getenv('PATH')),
            'AURACLE_TEST_TMPDIR': self.tempdir,
            'AURACLE_DEBUG': 'requests:{}'.format(self.requests_file),
        }

        cmdline = [
            os.path.join(self.build_dir, 'auracle'),
            '--baseurl', self.baseurl,
            '--color=never',
            '--chdir', self.tempdir,
        ] + list(args)

        p = subprocess.run(cmdline, env=env, capture_output=True)

        self._ProcessDebugOutput()

        return p

    def assertPkgbuildExists(self, pkgname, git=False):
        self.assertTrue(
                os.path.exists(os.path.join(self.tempdir, pkgname, 'PKGBUILD')))
        if git:
            self.assertTrue(
                    os.path.exists(os.path.join(self.tempdir, pkgname, '.git')))

def main():
    unittest.main()
<|endoftext|>


<commit_before>from django.core.urlresolvers import reverse
from django.conf import settings

from crashstats.crashstats.tests.test_views import BaseTestViews

class TestViews(BaseTestViews):

    def test_home(self):
        url = reverse('home:home', args=('WaterWolf',))
        response = self.client.get(url)
        assert response.status_code == 200
        assert 'WaterWolf Crash Data' in response.content
        assert 'WaterWolf 19.0' in response.content

    def test_home_product_without_featured_versions(self):
        url = reverse('home:home', args=('SeaMonkey',))
        response = self.client.get(url)
        assert response.status_code == 200
        assert 'SeaMonkey Crash Data' in response.content
        assert 'SeaMonkey 10.5' in response.content
        assert 'SeaMonkey 9.5' in response.content

    def test_homepage_redirect(self):
        response = self.client.get('/')
        assert response.status_code == 302
        destination = reverse('home:home', args=[settings.DEFAULT_PRODUCT])
        assert destination in response['Location']

    def test_homepage_products_redirect_without_versions(self):
        url = '/home/products/WaterWolf'
        # some legacy URLs have this
        url += '/versions/'

        redirect_code = settings.PERMANENT_LEGACY_REDIRECTS and 301 or 302
        destination = reverse('home:home', args=['WaterWolf'])

        response = self.client.get(url)
        assert response.status_code == redirect_code
        intermediate_dest = response['Location']

        response = self.client.get(intermediate_dest)
        assert response.status_code == redirect_code
        assert destination in response['Location'] == response['Location']
<commit_msg>Add test for missing_product view<commit_after>from django.core.urlresolvers import reverse
from django.conf import settings

from crashstats.crashstats.tests.test_views import BaseTestViews

class TestViews(BaseTestViews):

    def test_home(self):
        url = reverse('home:home', args=('WaterWolf',))
        response = self.client.get(url)
        assert response.status_code == 200
        assert 'WaterWolf Crash Data' in response.content
        assert 'WaterWolf 19.0' in response.content

    def test_home_product_missing(self):
        url = reverse('home:home', args=('PickleParty',))
        response = self.client.get(url)
        assert response.status_code == 404
        assert 'Missing product: PickleParty' in response.content

    def test_home_product_without_featured_versions(self):
        url = reverse('home:home', args=('SeaMonkey',))
        response = self.client.get(url)
        assert response.status_code == 200
        assert 'SeaMonkey Crash Data' in response.content
        assert 'SeaMonkey 10.5' in response.content
        assert 'SeaMonkey 9.5' in response.content

    def test_homepage_redirect(self):
        response = self.client.get('/')
        assert response.status_code == 302
        destination = reverse('home:home', args=[settings.DEFAULT_PRODUCT])
        assert destination in response['Location']

    def test_homepage_products_redirect_without_versions(self):
        url = '/home/products/WaterWolf'
        # some legacy URLs have this
        url += '/versions/'

        redirect_code = settings.PERMANENT_LEGACY_REDIRECTS and 301 or 302
        destination = reverse('home:home', args=['WaterWolf'])

        response = self.client.get(url)
        assert response.status_code == redirect_code
        intermediate_dest = response['Location']

        response = self.client.get(intermediate_dest)
        assert response.status_code == redirect_code
        assert destination in response['Location'] == response['Location']
<|endoftext|>


<commit_before>import os
from tools import check_cmd

import numpy as np
import tables

import visitors

def run_cyclus(cyclus, cwd, in_path, out_path):
    """Runs cyclus with various inputs and creates output databases
    """
    holdsrtn = [1]  # needed because nose does not send() to test generator
    # make sure the output target directory exists
    cmd = [cyclus, "-o", out_path, "--input-file", in_path]
    check_cmd(cmd, cwd, holdsrtn)

def compare_nondeterm(path1, path2):
    """Compares two Cyclus HDF5 databases assuming non-deterministic AgentIDs
    and TransactionIDs

    Returns True if both databases are the same, taking into account
            nondeterministic id assignments
    """
    v1 = visitors.HDF5RegressionVisitor(path1)
    v2 = visitors.HDF5RegressionVisitor(path2)
    return v1.walk() == v2.walk()

def compare_determ(path1, path2, verbose=False):
    """Compares two Cyclus HDF5 databases assuming deterministic AgentIDs and
    TransactionIDs

    Returns True if both databases are identical other than their SimIDs
    """
    dbs_same = True
    db_one = tables.open_file(path1, mode = "r")
    db_two = tables.open_file(path2, mode = "r")
    path_one = []
    path_two = []

    for node in db_one.walk_nodes(classname = "Table"):
        path_one.append(node._v_pathname)

    for node in db_two.walk_nodes(classname = "Table"):
        path_two.append(node._v_pathname)

    # Check if databases contain the same tables
    if not np.all(path_one == path_two):
        if verbose:
            print("The number or names of tables in databases are not the same.")
            print(path_one)
            print(path_two)
        # Close databases
        db_one.close()
        db_two.close()
        dbs_same = False
        return dbs_same

    paths = path_one

    for path in paths:
        data_one = db_one.get_node(path)[:]
        data_two = db_two.get_node(path)[:]
        names = []

        for name in data_one.dtype.names:
            if name != "SimID":
                names.append(name)

        data_one = data_one[names]
        data_two = data_two[names]

        if np.all(data_one == data_two):
            continue

        dbs_same = False
        if verbose:
            msg = ""
            msg += path.replace("/", "")
            msg += " table is different in the databases.\n"
            msg += determ_err_msg(names, data_one, data_two)
            print(msg)

    # Close databases
    db_one.close()
    db_two.close()
    return dbs_same

def determ_err_msg(names, data_one, data_two):
    """Returns a string describing the deterministic difference between two
    databases.
    """
    msg = ""
    # Investigation of the differences
    # check if the lengths are different
    if len(data_one) != len(data_two):
        msg += "Length mismatch: " + str(len(data_one)) + ", " + str(len(data_two))
    else:
        for name in names:
            column_one = data_one[name]
            column_two = data_two[name]
            # check if data types are the same
            if column_one.dtype != column_two.dtype:
                msg += "Datatypes in column " + name +" are different."
                msg += str(column_one.dtype)
                msg += str(column_two.dtype)
            elif not np.all(column_one == column_two):
                msg += "Column " + name
                diff = np.equal(column_one, column_two)
                # find indices and elements for numerical values
                indices = np.where(diff==False)
                # check if whole table is different
                if len(indices) == len(column_one):
                    msg += " is completely different"
                else:
                    # provide mismatch percentage
                    mismatch = 100*float(len(indices))/len(column_one)
                    msg += " has a mismatch of"
                    msg += " {0:.2f}".format(mismatch) + "% \n"
                    msg += "Indices of different objects are:\n"
                    msg += str(indices[0]) + "\n"
                    msg += "The different elements on these indices: \n"
                    msg += str(column_one[indices]) + "\n"
                    msg += str(column_two[indices]) + "\n"
    return msg
<commit_msg>updated comment in cyclus_tools<commit_after>import os
from tools import check_cmd

import numpy as np
import tables

import visitors

def run_cyclus(cyclus, cwd, in_path, out_path):
    """Runs cyclus with various inputs and creates output databases
    """
    holdsrtn = [1]  # needed because nose does not send() to test generator
    # make sure the output target directory exists
    cmd = [cyclus, "-o", out_path, "--input-file", in_path]
    check_cmd(cmd, cwd, holdsrtn)

def compare_nondeterm(path1, path2):
    """Compares two Cyclus HDF5 databases assuming non-deterministic AgentIDs
    and TransactionIDs.

    Returns
    -------
    rtn : bool
        True if both databases are the same, taking into account
        nondeterministic id assignments.
    """
    v1 = visitors.HDF5RegressionVisitor(path1)
    v2 = visitors.HDF5RegressionVisitor(path2)
    return v1.walk() == v2.walk()

def compare_determ(path1, path2, verbose=False):
    """Compares two Cyclus HDF5 databases assuming deterministic AgentIDs and
    TransactionIDs

    Returns
    -------
    rtn : bool
        True if both databases are identical other than their SimIDs
    """
    dbs_same = True
    db_one = tables.open_file(path1, mode = "r")
    db_two = tables.open_file(path2, mode = "r")
    path_one = []
    path_two = []

    for node in db_one.walk_nodes(classname = "Table"):
        path_one.append(node._v_pathname)

    for node in db_two.walk_nodes(classname = "Table"):
        path_two.append(node._v_pathname)

    # Check if databases contain the same tables
    if not np.all(path_one == path_two):
        if verbose:
            print("The number or names of tables in databases are not the same.")
            print(path_one)
            print(path_two)
        # Close databases
        db_one.close()
        db_two.close()
        dbs_same = False
        return dbs_same

    paths = path_one

    for path in paths:
        data_one = db_one.get_node(path)[:]
        data_two = db_two.get_node(path)[:]
        names = []

        for name in data_one.dtype.names:
            if name != "SimID":
                names.append(name)

        data_one = data_one[names]
        data_two = data_two[names]

        if np.all(data_one == data_two):
            continue

        dbs_same = False
        if verbose:
            msg = ""
            msg += path.replace("/", "")
            msg += " table is different in the databases.\n"
            msg += determ_err_msg(names, data_one, data_two)
            print(msg)

    # Close databases
    db_one.close()
    db_two.close()
    return dbs_same

def determ_err_msg(names, data_one, data_two):
    """Returns a string describing the deterministic difference between two
    databases.
    """
    msg = ""
    # Investigation of the differences
    # check if the lengths are different
    if len(data_one) != len(data_two):
        msg += "Length mismatch: " + str(len(data_one)) + ", " + str(len(data_two))
    else:
        for name in names:
            column_one = data_one[name]
            column_two = data_two[name]
            # check if data types are the same
            if column_one.dtype != column_two.dtype:
                msg += "Datatypes in column " + name +" are different."
                msg += str(column_one.dtype)
                msg += str(column_two.dtype)
            elif not np.all(column_one == column_two):
                msg += "Column " + name
                diff = np.equal(column_one, column_two)
                # find indices and elements for numerical values
                indices = np.where(diff==False)
                # check if whole table is different
                if len(indices) == len(column_one):
                    msg += " is completely different"
                else:
                    # provide mismatch percentage
                    mismatch = 100*float(len(indices))/len(column_one)
                    msg += " has a mismatch of"
                    msg += " {0:.2f}".format(mismatch) + "% \n"
                    msg += "Indices of different objects are:\n"
                    msg += str(indices[0]) + "\n"
                    msg += "The different elements on these indices: \n"
                    msg += str(column_one[indices]) + "\n"
                    msg += str(column_two[indices]) + "\n"
    return msg
<|endoftext|>