<commit_before>from ppb_vector import Vector2
from utils import angle_isclose, vectors
import pytest  # type: ignore
import math
from hypothesis import assume, given, note
import hypothesis.strategies as st

data = [
    (Vector2(1, 1), 90, Vector2(-1, 1)),
    (Vector2(1, 1), 180, Vector2(-1, -1)),
    (Vector2(1, 1), 360, Vector2(1, 1)),
    (Vector2(3, -20), 53, Vector2(17.77816, -9.64039)),
    (Vector2(math.pi, -1 * math.e), 30, Vector2(4.07984, -0.7833)),
    (Vector2(math.pi, math.e), 67, Vector2(-1.27467, 3.95397))
]

@pytest.mark.parametrize('input, degrees, expected', data)
def test_multiple_rotations(input, degrees, expected):
    assert input.rotate(degrees).isclose(expected)
    assert angle_isclose(input.angle(expected), degrees)

def test_for_exception():
    with pytest.raises(TypeError):
        Vector2('gibberish', 1).rotate(180)

@given(degree=st.floats(min_value=-360, max_value=360))
def test_trig_stability(degree):
    r = math.radians(degree)
    r_cos = math.cos(r)
    r_sin = math.sin(r)
    # Don't use exponents here. Multiplication is generally more stable.
    assert math.isclose(r_cos * r_cos + r_sin * r_sin, 1)

@given(
    initial=vectors(),
    angle=st.floats(min_value=-360, max_value=360),
)
def test_rotation_angle(initial, angle):
    assume(initial.length > 1e-5)
    rotated = initial.rotate(angle)
    note(f"Rotated: {rotated}")

    measured_angle = initial.angle(rotated)
    d = measured_angle - angle % 360
    note(f"Angle: {measured_angle} = {angle} + {d if d<180 else d-360}")
    assert angle_isclose(angle, measured_angle)

@given(
    increment=st.floats(min_value=1e-3, max_value=360),
    loops=st.integers(min_value=0)
)
def test_rotation_stability(increment, loops):
    initial = Vector2(1, 0)

    fellswoop = initial.rotate(increment * loops)
    note(f"One Fell Swoop: {fellswoop}")

    stepwise = initial
    for _ in range(loops):
        stepwise = stepwise.rotate(increment)
    note(f"Step-wise: {stepwise}")

    assert fellswoop.isclose(stepwise)

@given(
    initial=vectors(),
    angles=st.lists(st.floats(min_value=-360, max_value=360)),
)
def test_rotation_stability2(initial, angles):
    total_angle = sum(angles)
    fellswoop = initial.rotate(total_angle)
    note(f"One Fell Swoop: {fellswoop}")

    stepwise = initial
    for angle in angles:
        stepwise = stepwise.rotate(angle)
    note(f"Step-wise: {stepwise}")

    assert fellswoop.isclose(stepwise)

@given(
    a=vectors(max_magnitude=1e150), b=vectors(),
    l=st.floats(min_value=-1e150, max_value=1e150),
    angle=st.floats(min_value=-360, max_value=360),
)
def test_rotation_linearity(a, b, l, angle):
    inner = (l * a + b).rotate(angle)
    outer = l * a.rotate(angle) + b.rotate(angle)
    assert inner.isclose(outer)
<commit_msg>tests/rotate: Assert that rotation doesn't change the length<commit_after>from ppb_vector import Vector2
from utils import angle_isclose, vectors
import pytest  # type: ignore
import math
from hypothesis import assume, given, note
import hypothesis.strategies as st

data = [
    (Vector2(1, 1), 90, Vector2(-1, 1)),
    (Vector2(1, 1), 180, Vector2(-1, -1)),
    (Vector2(1, 1), 360, Vector2(1, 1)),
    (Vector2(3, -20), 53, Vector2(17.77816, -9.64039)),
    (Vector2(math.pi, -1 * math.e), 30, Vector2(4.07984, -0.7833)),
    (Vector2(math.pi, math.e), 67, Vector2(-1.27467, 3.95397))
]

@pytest.mark.parametrize('input, degrees, expected', data)
def test_multiple_rotations(input, degrees, expected):
    assert input.rotate(degrees).isclose(expected)
    assert angle_isclose(input.angle(expected), degrees)

def test_for_exception():
    with pytest.raises(TypeError):
        Vector2('gibberish', 1).rotate(180)

@given(degree=st.floats(min_value=-360, max_value=360))
def test_trig_stability(degree):
    r = math.radians(degree)
    r_cos = math.cos(r)
    r_sin = math.sin(r)
    # Don't use exponents here. Multiplication is generally more stable.
    assert math.isclose(r_cos * r_cos + r_sin * r_sin, 1)

@given(
    initial=vectors(),
    angle=st.floats(min_value=-360, max_value=360),
)
def test_rotation_angle(initial, angle):
    assume(initial.length > 1e-5)
    rotated = initial.rotate(angle)
    note(f"Rotated: {rotated}")

    measured_angle = initial.angle(rotated)
    d = measured_angle - angle % 360
    note(f"Angle: {measured_angle} = {angle} + {d if d<180 else d-360}")
    assert angle_isclose(angle, measured_angle)

@given(
    increment=st.floats(min_value=1e-3, max_value=360),
    loops=st.integers(min_value=0)
)
def test_rotation_stability(increment, loops):
    initial = Vector2(1, 0)

    fellswoop = initial.rotate(increment * loops)
    note(f"One Fell Swoop: {fellswoop}")

    stepwise = initial
    for _ in range(loops):
        stepwise = stepwise.rotate(increment)
    note(f"Step-wise: {stepwise}")

    assert fellswoop.isclose(stepwise)
    assert math.isclose(fellswoop.length, initial.length)

@given(
    initial=vectors(),
    angles=st.lists(st.floats(min_value=-360, max_value=360)),
)
def test_rotation_stability2(initial, angles):
    total_angle = sum(angles)
    fellswoop = initial.rotate(total_angle)
    note(f"One Fell Swoop: {fellswoop}")

    stepwise = initial
    for angle in angles:
        stepwise = stepwise.rotate(angle)
    note(f"Step-wise: {stepwise}")

    assert fellswoop.isclose(stepwise)
    assert math.isclose(fellswoop.length, initial.length)

@given(
    a=vectors(max_magnitude=1e150), b=vectors(),
    l=st.floats(min_value=-1e150, max_value=1e150),
    angle=st.floats(min_value=-360, max_value=360),
)
def test_rotation_linearity(a, b, l, angle):
    inner = (l * a + b).rotate(angle)
    outer = l * a.rotate(angle) + b.rotate(angle)
    assert inner.isclose(outer)
<|endoftext|>


<commit_before># Copyright (c) 2015 Nicolas JOUANIN
#
# See the file license.txt for copying permission.
import asyncio
from hbmqtt.utils import (
    bytes_to_hex_str,
    bytes_to_int,
    read_or_raise,
)
from hbmqtt.message import FixedHeader, MessageType
from hbmqtt.streams.errors import StreamException, NoDataException

class FixedHeaderException(StreamException):
    pass

class FixedHeaderStream:
    def __init__(self):
        pass

    def decode(self, reader) -> FixedHeader:
        b1 = yield from read_or_raise(reader, 1)
        msg_type = FixedHeaderStream.get_message_type(b1)
        if msg_type is MessageType.RESERVED_0 or msg_type is MessageType.RESERVED_15:
            raise FixedHeaderException("Usage of control packet type %s is forbidden" % msg_type)
        flags = FixedHeaderStream.get_flags(b1)

        remain_length = yield from self.decode_remaining_length(reader)
        return FixedHeader(msg_type, flags, remain_length)

    @staticmethod
    def get_message_type(byte):
        byte_type = (bytes_to_int(byte) & 0xf0) >> 4
        return MessageType(byte_type)

    @staticmethod
    def get_flags(data):
        byte = bytes_to_int(data)
        return byte & 0x0f

    @asyncio.coroutine
    def decode_remaining_length(self, reader):
        multiplier = 1
        value = 0
        length_bytes = b''
        while True:
            encoded_byte = yield from read_or_raise(reader, 1)
            length_bytes += encoded_byte
            int_byte = bytes_to_int(encoded_byte)
            value += (int_byte & 0x7f) * multiplier
            if (int_byte & 0x80) == 0:
                break
            else:
                multiplier *= 128
                if multiplier > 128*128*128:
                    raise FixedHeaderException("Invalid remaining length bytes:%s" % bytes_to_hex_str(length_bytes))
        return value
<commit_msg>Add some comments<commit_after># Copyright (c) 2015 Nicolas JOUANIN
#
# See the file license.txt for copying permission.
import asyncio
from hbmqtt.utils import (
    bytes_to_hex_str,
    bytes_to_int,
    read_or_raise,
)
from hbmqtt.message import FixedHeader, MessageType
from hbmqtt.streams.errors import StreamException, NoDataException

class FixedHeaderException(StreamException):
    pass

class FixedHeaderStream:
    def __init__(self):
        pass

    @asyncio.coroutine
    def decode(self, reader) -> FixedHeader:
        """
        Decode MQTT message fixed header from stream
        :param reader: Stream to read
        :return: FixedHeader instance
        """
        b1 = yield from read_or_raise(reader, 1)
        msg_type = FixedHeaderStream.get_message_type(b1)
        if msg_type is MessageType.RESERVED_0 or msg_type is MessageType.RESERVED_15:
            raise FixedHeaderException("Usage of control packet type %s is forbidden" % msg_type)
        flags = FixedHeaderStream.get_flags(b1)

        remain_length = yield from self.decode_remaining_length(reader)
        return FixedHeader(msg_type, flags, remain_length)

    @staticmethod
    def get_message_type(byte):
        byte_type = (bytes_to_int(byte) & 0xf0) >> 4
        return MessageType(byte_type)

    @staticmethod
    def get_flags(data):
        byte = bytes_to_int(data)
        return byte & 0x0f

    @asyncio.coroutine
    def decode_remaining_length(self, reader):
        """
        Decode message length according to MQTT specifications
        :param reader:
        :return:
        """
        multiplier = 1
        value = 0
        length_bytes = b''
        while True:
            encoded_byte = yield from read_or_raise(reader, 1)
            length_bytes += encoded_byte
            int_byte = bytes_to_int(encoded_byte)
            value += (int_byte & 0x7f) * multiplier
            if (int_byte & 0x80) == 0:
                break
            else:
                multiplier *= 128
                if multiplier > 128*128*128:
                    raise FixedHeaderException("Invalid remaining length bytes:%s" % bytes_to_hex_str(length_bytes))
        return value
<|endoftext|>


<commit_before>from math import ceil, log2
import csv

from termcolor import colored
import numpy as np
import matplotlib.pyplot as plot
import matplotlib.gridspec as gridspec
import matplotlib.cm as cm
from matplotlib.widgets import Slider

import data_handling as dat

with open("output.csv", newline="") as csv_file:
    print("Processing CSV... (This might take a few seconds.)")
    lines = csv.reader(csv_file, delimiter=";", quoting=csv.QUOTE_NONNUMERIC)
    lines = sorted(sorted(lines, key=lambda x: x[1]), key=lambda x: x[0])
    v_sd_range = np.asarray(sorted(list(set([line[1] for line in lines]))))
    v_g_range = np.asarray(sorted(list(set([line[0] for line in lines]))))
    voltage_area = (v_sd_range, v_g_range)
    iw_list = []
    for line in lines:
        current = line[2]
        weights = line[3:]
        iw_list.append((current, weights))

# Define some colors in the format required by matplotlib.
red = np.array([[1, 0, 0]])
black = np.array([[0, 0, 0]])
white = np.array([[1, 1, 1]])

whitespace = "".join(["\n" for _ in range(100)])

def i_sd(v_g=0):
    gs = gridspec.GridSpec(1, 1)
    axes = plot.subplot(gs[0])
    data = dat.get_i_vs_v_sd(iw_list, v_g, voltage_area)
    axes.plot(v_sd_range, data)

    axes.set_xlim([v_sd_range[0], v_sd_range[-1]])

    axes.set_title(r"at $V_\mathrm{g} = " + "%.2f" % v_g + "\ \mathrm{V}$")
    axes.set_ylabel(r"$\sigma / \mathrm{arb.}$")
    axes.set_xlabel(r"$V_\mathrm{sd} / \mathrm{V}$")

    plot.show(block=False)

def diff_cond(v_g=0):
    gs = gridspec.GridSpec(1, 1)
    axes = plot.subplot(gs[0])
    data = dat.get_diff_conductance_vs_v_sd(iw_list, v_g, voltage_area)
    axes.plot(v_sd_range, data)

    axes.set_xlim([v_sd_range[0], v_sd_range[-1]])

    axes.set_title(r"at $V_\mathrm{g} = " + "%.2f" % v_g + "\ \mathrm{V}$")
    axes.set_ylabel(r"$\frac{\mathrm{d}\sigma} "
                    + r"{\mathrm{d}V_\mathrm{sd}} / \mathrm{arb.}$")
    axes.set_xlabel(r"$V_\mathrm{sd} / \mathrm{V}$")

    plot.show(block=False)

def mean_occupation(v_g=0):
    gs = gridspec.GridSpec(1, 1)
    axes = plot.subplot(gs[0])
    data = dat.get_mean_occupation_vs_v_sd(iw_list, v_g, voltage_area)
    axes.plot(v_sd_range, data)

    axes.set_xlim([v_sd_range[0], v_sd_range[-1]])

    axes.set_title(r"at $V_\mathrm{g} = " + "%.2f" % v_g + "\ \mathrm{V}$")
    axes.set_ylabel(r"Mean dot occupation number")
    axes.set_xlabel(r"$V_\mathrm{sd} / \mathrm{V}$")

    plot.show(block=False)

def heatmap(axes=None):
    if axes is None:
        show = True
        gs = gridspec.GridSpec(1, 2, width_ratios=[10, 1])
        heatmap_axes = plot.subplot(gs[0])
        colorbar_axes = plot.subplot(gs[1])
    else:
        show = False
        heatmap_axes, colorbar_axes = axes

    current_function, extent = dat.get_plottable_diff_conductance_in_v_space(
        iw_list, voltage_area)
    heatmap_ = heatmap_axes.imshow(current_function, extent=extent,
                                   interpolation="nearest", aspect="auto",
                                   cmap=cm.rainbow)
    plot.colorbar(heatmap_, cax=colorbar_axes)

    heatmap_axes.set_xlim([v_sd_range[0], v_sd_range[-1]])
    heatmap_axes.set_ylim([v_g_range[0], v_g_range[-1]])

    heatmap_axes.set_xlabel(r"$V_{sd}/\mathrm{V}$")
    heatmap_axes.set_ylabel(r"$V_\mathrm{g}/\mathrm{V}$")
    colorbar_axes.set_title(
        r"$\frac{\partial I}{\partial V_\mathrm{sd}} /\mathrm{arb.\ units}$")

    if show:
        plot.show(block=False)

def pretty_bin(number, max_number):
    width = ceil(log2(max_number))
    return "".join([colored("*", "white") if dat.bit(number, index)
                    else colored("|", "red")
                    for index in range(width)])

def ui(v_sd=0, v_g=0):
    """Display a graphical user interface for data exploration."""

    # Define the layout of the UI and add "plots" to UI regions.
    gs = gridspec.GridSpec(
        6, 3,
        height_ratios=[10, 2, 10, 2, 1, 1], width_ratios=[20, 1, 2])
    heatmap_axes = plot.subplot(gs[0])
    colorbar_axes = plot.subplot(gs[2])
    line_plot_axes = plot.subplot(gs[6])
    v_sd_slider_axes = plot.subplot(gs[12])
    v_g_slider_axes = plot.subplot(gs[15])

    # Slider UI elements
    # ------------------
    v_sd_slider = Slider(v_sd_slider_axes, r"$V_\mathrm{sd}/\mathrm{V}$",
                         v_sd_range[0], v_sd_range[-1],
                         valinit=v_sd)
    v_g_slider = Slider(
        v_g_slider_axes, r"$V_\mathrm{g}/\mathrm{V}$",
        v_g_range[0], v_g_range[-1],
        valinit=v_g)

    # Use an object to store interactive voltages.
    # This makes it simpler to keep track of slider changes persistently.
    class Voltages:
        sd = v_sd
        g = v_g

    v = Voltages()

    # Conductance heat map
    # --------------------
    heatmap(tuple([heatmap_axes, colorbar_axes]))

    # We will add a small cross marking the current position in v_g-v_sd space
    # to the heatmap plot. To keep track of its position, we use the following
    # object.
    class PointAnnotation():
        point = heatmap_axes.scatter(v_sd, v_g, marker="+")

    annotation = PointAnnotation()

    def replot_point_annotation(v_sd_, v_g_):
        annotation.point.remove()
        annotation.point = heatmap_axes.scatter(v_sd_, v_g_, marker="+")

    # We also add lines annotating values of v_sd and v_g on the two large
    # plots. v_g_line is persistent and has its y_data updated in redraw().
    # There is no persistent equivalent v_sd_line, because the line plot is
    # completely erased and replotted on each redraw() call; we will make a
    # new axvline each time.
    line_plot_axes.axvline(x=v.sd)
    v_g_line = heatmap_axes.axhline(y=v.g)

    # Labels
    # ------
    def label_figures():
        line_plot_axes.set_xlabel(r"$V_\mathrm{sd}/\mathrm{V}$")
        line_plot_axes.set_ylabel(r"Current $I / \mathrm{arb.}$")

    # Slider update actions
    # ---------------------
    def redraw():
        # Move annotations.
        replot_point_annotation(v.sd, v.g)
        v_g_line.set_ydata(v.g)

        # Retrieve from memory and plot current.
        i_vs_v_sd = dat.get_i_vs_v_sd(iw_list, v.g, voltage_area)
        line_plot_axes.clear()
        line_plot_axes.plot(v_sd_range,
                            [current for current in i_vs_v_sd],
                            "black")
        line_plot_axes.set_xlim([v_sd_range[0], v_sd_range[-1]])
        line_plot_axes.axvline(x=v.sd)

        current, weights = dat.get_iw_tuple(iw_list, v.sd, v.g, voltage_area)

        print(whitespace)

        v.sd, v.g = dat.get_voltage_pair_from_index(
            dat.get_index_from_voltage_pair(v.sd, v.g, voltage_area),
            voltage_area)

        for index, weight in sorted([(index, weight)
                                     for index, weight in enumerate(weights)],
                                    key=lambda x: x[1], reverse=False):

            if weight > 0.0001:
                weight_bar = "".join([colored("=", "green") if point < weight
                                      else " "
                                      for point
                                      in np.linspace(0, 1-1e-10, 40)])

                print(" " + pretty_bin(index, len(weights)),
                      "%.3f" % weight, weight_bar)

        print("\nv_g/V =", "%.3f" % v.g, "; v_sd/V =", "%.3f" % v.sd)
        print("mean occupation =", "%.3f" % dat.mean_occupation(weights))
        print("current =", "%.3f" % (current), "arb. units")

        print("\n>>> ", end="")

        label_figures()

    def update_v_sd(v_sd_):
        v.sd = v_sd_
        redraw()

    def update_v_g(v_g_):
        v.g = v_g_
        redraw()

    # Set sliders to listen for clicks.
    v_sd_slider.on_changed(update_v_sd)
    v_g_slider.on_changed(update_v_g)

    # Draw with default v_g, v_sd values.
    redraw()

    # Launch the matplotlib window.
    plot.show(block=False)

print("Try ui(), or heatmap().")
<commit_msg>UI color change<commit_after>from math import ceil, log2
import csv

from termcolor import colored
import numpy as np
import matplotlib.pyplot as plot
import matplotlib.gridspec as gridspec
import matplotlib.cm as cm
from matplotlib.widgets import Slider

import data_handling as dat

with open("output.csv", newline="") as csv_file:
    print("Processing CSV... (This might take a few seconds.)")
    lines = csv.reader(csv_file, delimiter=";", quoting=csv.QUOTE_NONNUMERIC)
    lines = sorted(sorted(lines, key=lambda x: x[1]), key=lambda x: x[0])
    v_sd_range = np.asarray(sorted(list(set([line[1] for line in lines]))))
    v_g_range = np.asarray(sorted(list(set([line[0] for line in lines]))))
    voltage_area = (v_sd_range, v_g_range)
    iw_list = []
    for line in lines:
        current = line[2]
        weights = line[3:]
        iw_list.append((current, weights))

# Define some colors in the format required by matplotlib.
red = np.array([[1, 0, 0]])
black = np.array([[0, 0, 0]])
white = np.array([[1, 1, 1]])

whitespace = "".join(["\n" for _ in range(100)])

def i_sd(v_g=0):
    gs = gridspec.GridSpec(1, 1)
    axes = plot.subplot(gs[0])
    data = dat.get_i_vs_v_sd(iw_list, v_g, voltage_area)
    axes.plot(v_sd_range, data)

    axes.set_xlim([v_sd_range[0], v_sd_range[-1]])

    axes.set_title(r"at $V_\mathrm{g} = " + "%.2f" % v_g + "\ \mathrm{V}$")
    axes.set_ylabel(r"$\sigma / \mathrm{arb.}$")
    axes.set_xlabel(r"$V_\mathrm{sd} / \mathrm{V}$")

    plot.show(block=False)

def diff_cond(v_g=0):
    gs = gridspec.GridSpec(1, 1)
    axes = plot.subplot(gs[0])
    data = dat.get_diff_conductance_vs_v_sd(iw_list, v_g, voltage_area)
    axes.plot(v_sd_range, data)

    axes.set_xlim([v_sd_range[0], v_sd_range[-1]])

    axes.set_title(r"at $V_\mathrm{g} = " + "%.2f" % v_g + "\ \mathrm{V}$")
    axes.set_ylabel(r"$\frac{\mathrm{d}\sigma} "
                    + r"{\mathrm{d}V_\mathrm{sd}} / \mathrm{arb.}$")
    axes.set_xlabel(r"$V_\mathrm{sd} / \mathrm{V}$")

    plot.show(block=False)

def mean_occupation(v_g=0):
    gs = gridspec.GridSpec(1, 1)
    axes = plot.subplot(gs[0])
    data = dat.get_mean_occupation_vs_v_sd(iw_list, v_g, voltage_area)
    axes.plot(v_sd_range, data)

    axes.set_xlim([v_sd_range[0], v_sd_range[-1]])

    axes.set_title(r"at $V_\mathrm{g} = " + "%.2f" % v_g + "\ \mathrm{V}$")
    axes.set_ylabel(r"Mean dot occupation number")
    axes.set_xlabel(r"$V_\mathrm{sd} / \mathrm{V}$")

    plot.show(block=False)

def heatmap(axes=None):
    if axes is None:
        show = True
        gs = gridspec.GridSpec(1, 2, width_ratios=[10, 1])
        heatmap_axes = plot.subplot(gs[0])
        colorbar_axes = plot.subplot(gs[1])
    else:
        show = False
        heatmap_axes, colorbar_axes = axes

    current_function, extent = dat.get_plottable_diff_conductance_in_v_space(
        iw_list, voltage_area)
    heatmap_ = heatmap_axes.imshow(current_function, extent=extent,
                                   interpolation="nearest", aspect="auto",
                                   cmap=cm.binary)
    plot.colorbar(heatmap_, cax=colorbar_axes)

    heatmap_axes.set_xlim([v_sd_range[0], v_sd_range[-1]])
    heatmap_axes.set_ylim([v_g_range[0], v_g_range[-1]])

    heatmap_axes.set_xlabel(r"$V_{sd}/\mathrm{V}$")
    heatmap_axes.set_ylabel(r"$V_\mathrm{g}/\mathrm{V}$")
    colorbar_axes.set_title(
        r"$\frac{\partial I}{\partial V_\mathrm{sd}} /\mathrm{arb.\ units}$")

    if show:
        plot.show(block=False)

def pretty_bin(number, max_number):
    width = ceil(log2(max_number))
    return "".join([colored("*", "white") if dat.bit(number, index)
                    else colored("|", "red")
                    for index in range(width)])

def ui(v_sd=0, v_g=0):
    """Display a graphical user interface for data exploration."""

    # Define the layout of the UI and add "plots" to UI regions.
    gs = gridspec.GridSpec(
        6, 3,
        height_ratios=[10, 2, 10, 2, 1, 1], width_ratios=[20, 1, 2])
    heatmap_axes = plot.subplot(gs[0])
    colorbar_axes = plot.subplot(gs[2])
    line_plot_axes = plot.subplot(gs[6])
    v_sd_slider_axes = plot.subplot(gs[12])
    v_g_slider_axes = plot.subplot(gs[15])

    # Slider UI elements
    # ------------------
    v_sd_slider = Slider(v_sd_slider_axes, r"$V_\mathrm{sd}/\mathrm{V}$",
                         v_sd_range[0], v_sd_range[-1],
                         valinit=v_sd)
    v_g_slider = Slider(
        v_g_slider_axes, r"$V_\mathrm{g}/\mathrm{V}$",
        v_g_range[0], v_g_range[-1],
        valinit=v_g)

    # Use an object to store interactive voltages.
    # This makes it simpler to keep track of slider changes persistently.
    class Voltages:
        sd = v_sd
        g = v_g

    v = Voltages()

    # Conductance heat map
    # --------------------
    heatmap(tuple([heatmap_axes, colorbar_axes]))

    # We will add a small cross marking the current position in v_g-v_sd space
    # to the heatmap plot. To keep track of its position, we use the following
    # object.
    class PointAnnotation():
        point = heatmap_axes.scatter(v_sd, v_g, marker="+")

    annotation = PointAnnotation()

    def replot_point_annotation(v_sd_, v_g_):
        annotation.point.remove()
        annotation.point = heatmap_axes.scatter(v_sd_, v_g_, marker="+")

    # We also add lines annotating values of v_sd and v_g on the two large
    # plots. v_g_line is persistent and has its y_data updated in redraw().
    # There is no persistent equivalent v_sd_line, because the line plot is
    # completely erased and replotted on each redraw() call; we will make a
    # new axvline each time.
    line_plot_axes.axvline(x=v.sd)
    v_g_line = heatmap_axes.axhline(y=v.g)

    # Labels
    # ------
    def label_figures():
        line_plot_axes.set_xlabel(r"$V_\mathrm{sd}/\mathrm{V}$")
        line_plot_axes.set_ylabel(r"Current $I / \mathrm{arb.}$")

    # Slider update actions
    # ---------------------
    def redraw():
        # Move annotations.
        replot_point_annotation(v.sd, v.g)
        v_g_line.set_ydata(v.g)

        # Retrieve from memory and plot current.
        i_vs_v_sd = dat.get_i_vs_v_sd(iw_list, v.g, voltage_area)
        line_plot_axes.clear()
        line_plot_axes.plot(v_sd_range,
                            [current for current in i_vs_v_sd],
                            "black")
        line_plot_axes.set_xlim([v_sd_range[0], v_sd_range[-1]])
        line_plot_axes.axvline(x=v.sd)

        current, weights = dat.get_iw_tuple(iw_list, v.sd, v.g, voltage_area)

        print(whitespace)

        v.sd, v.g = dat.get_voltage_pair_from_index(
            dat.get_index_from_voltage_pair(v.sd, v.g, voltage_area),
            voltage_area)

        for index, weight in sorted([(index, weight)
                                     for index, weight in enumerate(weights)],
                                    key=lambda x: x[1], reverse=False):

            if weight > 0.0001:
                weight_bar = "".join([colored("=", "green") if point < weight
                                      else " "
                                      for point
                                      in np.linspace(0, 1 - 1e-10, 40)])

                print(" " + pretty_bin(index, len(weights)),
                      "%.3f" % weight, weight_bar)

        print("\nv_g/V =", "%.3f" % v.g, "; v_sd/V =", "%.3f" % v.sd)
        print("mean occupation =", "%.3f" % dat.mean_occupation(weights))
        print("current =", "%.3f" % (current), "arb. units")

        print("\n>>> ", end="")

        label_figures()

    def update_v_sd(v_sd_):
        v.sd = v_sd_
        redraw()

    def update_v_g(v_g_):
        v.g = v_g_
        redraw()

    # Set sliders to listen for clicks.
    v_sd_slider.on_changed(update_v_sd)
    v_g_slider.on_changed(update_v_g)

    # Draw with default v_g, v_sd values.
    redraw()

    # Launch the matplotlib window.
    plot.show(block=False)

print("Try ui(), or heatmap().")
<|endoftext|>


<commit_before><commit_msg>Fix typos in core.compress_pdf<commit_after><|endoftext|>


<commit_before><commit_msg>Config pelicanconfig locale for travis.<commit_after><|endoftext|>


<commit_before>"""
Module that contains the command line app.

Why does this file exist, and why not put this in __main__?

  You might be tempted to import things from __main__ later, but that will cause
  problems: the code will get executed twice:

  - When you run `python -mtwelve_tone` python will execute
    ``__main__.py`` as a script. That means there won't be any
    ``twelve_tone.__main__`` in ``sys.modules``.
  - When you import __main__ it will get executed again (as a module) because
    there's no ``twelve_tone.__main__`` in ``sys.modules``.

  Also see (1) from http://click.pocoo.org/5/setuptools/#setuptools-integration
"""
import click

from twelve_tone.composer import Composer

@click.command()
def main():
    c = Composer()
    c.compose()
    click.echo(c.get_melody())
<commit_msg>Closes #9 Added cli arguments to click to specify row and column number for selecting row or column tone<commit_after>"""
Module that contains the command line app.

Why does this file exist, and why not put this in __main__?

  You might be tempted to import things from __main__ later, but that will cause
  problems: the code will get executed twice:

  - When you run `python -mtwelve_tone` python will execute
    ``__main__.py`` as a script. That means there won't be any
    ``twelve_tone.__main__`` in ``sys.modules``.
  - When you import __main__ it will get executed again (as a module) because
    there's no ``twelve_tone.__main__`` in ``sys.modules``.

  Also see (1) from http://click.pocoo.org/5/setuptools/#setuptools-integration
"""
import click

from twelve_tone.composer import Composer

@click.command()
@click.option('--row', '-r', default=0, help='Row to use as row tone')
@click.option('--column', '-c', default=0, help='Column to use as column tone')
def main(row, column):
    c = Composer()
    c.compose()
    if row < 0 or column < 0:
      click.echo("Invalid row or column arguments.")
      exit(1)
    elif row >= c.matrix.shape[0]:
      click.echo("Row number exceeds melody row count.")
      exit(1)
    elif column >= c.matrix.shape[1]:
      click.echo("Column number exceeds melody column count.")
      exit(1)
    click.echo(c.get_melody(row=row, column=column))
<|endoftext|>


<commit_before><commit_msg>memperbaiki urutan data dalam tabel organisasi<commit_after><|endoftext|>


<commit_before>#!/usr/bin/env python3

# Forward messages from LoRa devices to AWS IoT, and forward AWS IoT messages to LoRa devices.

import time
import datetime
import ssl
import json
import paho.mqtt.client as mqtt
import lora_interface

# AWS IoT Device l001 to l255 <--> LoRa address 1 to 255
# AWS IoT Device l000 <--> LoRa broadcast (address 0)
# Address 1 is the LoRa Gateway
deviceName = "l001"

# Public certificate of our Raspberry Pi, as provided by AWS IoT.
deviceCertificate = "tp-iot-certificate.pem.crt"
# Private key of our Raspberry Pi, as provided by AWS IoT.
devicePrivateKey = "tp-iot-private.pem.key"
# Root certificate to authenticate AWS IoT when we connect to their server.
awsCert = "aws-iot-rootCA.crt"

isConnected = False

# This is the main logic of the program.  We connect to AWS IoT via MQTT, send sensor data periodically to AWS IoT,
# and handle any actuation commands received from AWS IoT.
def main():
    global isConnected
    # Create an MQTT client for connecting to AWS IoT via MQTT.
    client = mqtt.Client("lora_gateway")  # Client ID must be unique because AWS will disconnect any duplicates.
    client.on_connect = on_connect  # When connected, call on_connect.
    client.on_message = on_message  # When message received, call on_message.
    client.on_log = on_log  # When logging debug messages, call on_log.

    # Set the certificates and private key for connecting to AWS IoT.  TLS 1.2 is mandatory for AWS IoT and is supported
    # only in Python 3.4 and later, compiled with OpenSSL 1.0.1 and later.
    client.tls_set(awsCert, deviceCertificate, devicePrivateKey, ssl.CERT_REQUIRED, ssl.PROTOCOL_TLSv1_2)

    # Connect to AWS IoT server.  Use AWS command line "aws iot describe-endpoint" to get the address.
    print("Connecting to AWS IoT...")
    client.connect("A1P01IYM2DOZA0.iot.us-west-2.amazonaws.com", 8883, 60)

    # Start a background thread to process the MQTT network commands concurrently, including auto-reconnection.
    client.loop_start()

    # Setup the LoRa connection.  TODO: Check status
    status = lora_interface.setupLoRa()
    time.sleep(1)

    # Loop forever.
    while True:
        try:
            # If we are not connected yet to AWS IoT, wait 1 second and try again.
            if not isConnected:
                time.sleep(1)
                continue

            # Read a LoRa message, which contains the device state.  TODO: Check status.
            msg = lora_interface.readLoRaMessage()
            status = lora_interface.getLoRaStatus()

            # If no message available, wait 1 second and try again.
            if len(msg) == 0:
                time.sleep(1)
                continue

            # Assume msg contains a JSON string with sensor names and values like
            # {
            #    "temperature": 27.3,
            #    "humidity": 88
            # }
            # TODO: Comment this section.
            msg = '''{
                "temperature": 27.3,
                "humidity": 88
            }'''
            device_state = json.loads(msg)

            # Send the reported device state to AWS IoT.
            payload = {
                "state": {
                    "reported": device_state
                }
            }
            print("Sending sensor data to AWS IoT...\n" +
                  json.dumps(payload, indent=4, separators=(',', ': ')))

            # Publish our sensor data to AWS IoT via the MQTT topic, also known as updating our "Thing Shadow".
            # TODO: Derive AWS IoT device from LoRa address.
            client.publish("$aws/things/" + deviceName + "/shadow/update", json.dumps(payload))
            print("Sent to AWS IoT")

            # Wait 30 seconds before sending the next message.
            time.sleep(30)

        except KeyboardInterrupt:
            # Stop the program when we press Ctrl-C.
            break
        except Exception as e:
            # For all other errors, we wait a while and resume.
            print("Exception: " + str(e))
            time.sleep(10)
            continue

# This is called when we are connected to AWS IoT via MQTT.
# We subscribe for notifications of desired state updates.
def on_connect(client, userdata, flags, rc):
    global isConnected
    isConnected = True
    print("Connected to AWS IoT")
    # Subscribe to our MQTT topic so that we will receive notifications of updates.
    topic = "$aws/things/" + deviceName + "/shadow/update/accepted"
    print("Subscribing to MQTT topic " + topic)
    client.subscribe(topic)

# This is called when we receive a subscription notification from AWS IoT.
# If this is an actuation command, we execute it.
def on_message(client, userdata, msg):
    # Convert the JSON payload to a Python dictionary.
    # The payload is in binary format so we need to decode as UTF-8.
    payload2 = json.loads(msg.payload.decode("utf-8"))
    print("Received message, topic: " + msg.topic + ", payload:\n" +
          json.dumps(payload2, indent=4, separators=(',', ': ')))

    # If there is a desired state in this message, then we actuate, e.g. if we see "led=on", we switch on the LED.
    if payload2.get("state") is not None and payload2["state"].get("desired") is not None:
        # Get the desired state and loop through all attributes inside.
        desired_state = payload2["state"]["desired"]
        for attribute in desired_state:
            # We handle the attribute and desired value by actuating.
            value = desired_state.get(attribute)
            actuate(client, attribute, value)

# Control my actuators based on the specified attribute and value, e.g. "led=on" will switch on my LED.
def actuate(client, attribute, value):
    if attribute == "timestamp":
        # Ignore the timestamp attribute, it's only for info.
        return
    print("Setting " + attribute + " to " + value + "...")
    if attribute == "led":
        # We actuate the LED for "on", "off" or "flash1".
        if value == "on":
            # Switch on LED.
            grovepi.digitalWrite(led, 1)
            send_reported_state(client, "led", "on")
            return
        elif value == "off":
            # Switch off LED.
            grovepi.digitalWrite(led, 0)
            send_reported_state(client, "led", "off")
            return
        elif value == "flash1":
            # Switch on LED, wait 1 second, switch it off.
            grovepi.digitalWrite(led, 1)
            send_reported_state(client, "led", "on")
            time.sleep(1)

            grovepi.digitalWrite(led, 0)
            send_reported_state(client, "led", "off")
            time.sleep(1)
            return
    # Show an error if attribute or value are incorrect.
    print("Error: Don't know how to set " + attribute + " to " + value)

# Send the reported state of our actuator tp AWS IoT after it has been triggered, e.g. "led": "on".
def send_reported_state(client, attribute, value):
    # Prepare our sensor data in JSON format.
    payload = {
        "state": {
            "reported": {
                attribute: value,
                "timestamp": datetime.datetime.now().isoformat()
            }
        }
    }
    print("Sending sensor data to AWS IoT...\n" +
          json.dumps(payload, indent=4, separators=(',', ': ')))

    # Publish our sensor data to AWS IoT via the MQTT topic, also known as updating our "Thing Shadow".
    client.publish("$aws/things/" + deviceName + "/shadow/update", json.dumps(payload))
    print("Sent to AWS IoT")

# Print out log messages for tracing.
def on_log(client, userdata, level, buf):
    print("Log: " + buf)

# Start the main program.
main()
<commit_msg>Complete build of LoRa gateway<commit_after>#!/usr/bin/env python3
# Forward messages from LoRa devices to AWS IoT, and forward AWS IoT messages to LoRa devices.

import time
import datetime
import ssl
import json
import paho.mqtt.client as mqtt
import lora_interface

# AWS IoT Device l001 to l255 <--> LoRa address 1 to 255
# AWS IoT Device l000 <--> LoRa broadcast (address 0)
# Address 1 is the LoRa Gateway
gateway_address = 1
min_device_address = 2
max_device_address = 255

# Public certificate of our Raspberry Pi, as provided by AWS IoT.
deviceCertificate = "tp-iot-certificate.pem.crt"
# Private key of our Raspberry Pi, as provided by AWS IoT.
devicePrivateKey = "tp-iot-private.pem.key"
# Root certificate to authenticate AWS IoT when we connect to their server.
awsCert = "aws-iot-rootCA.crt"

isConnected = False

# This is the main logic of the program.  We connect to AWS IoT via MQTT, send sensor data periodically to AWS IoT,
# and handle any actuation commands received from AWS IoT.
def main():
    global isConnected
    # Create an MQTT client for connecting to AWS IoT via MQTT.
    client = mqtt.Client("lora_gateway")  # Client ID must be unique because AWS will disconnect any duplicates.
    client.on_connect = on_connect  # When connected, call on_connect.
    client.on_message = on_message  # When message received, call on_message.
    client.on_log = on_log  # When logging debug messages, call on_log.

    # Set the certificates and private key for connecting to AWS IoT.  TLS 1.2 is mandatory for AWS IoT and is supported
    # only in Python 3.4 and later, compiled with OpenSSL 1.0.1 and later.
    client.tls_set(awsCert, deviceCertificate, devicePrivateKey, ssl.CERT_REQUIRED, ssl.PROTOCOL_TLSv1_2)

    # Connect to AWS IoT server.  Use AWS command line "aws iot describe-endpoint" to get the address.
    print("Connecting to AWS IoT...")
    client.connect("A1P01IYM2DOZA0.iot.us-west-2.amazonaws.com", 8883, 60)

    # Start a background thread to process the MQTT network commands concurrently, including auto-reconnection.
    client.loop_start()

    # Setup the LoRa connection.  TODO: Check status
    status = lora_interface.setupLoRa()
    time.sleep(1)

    # Loop forever.
    while True:
        try:
            # If we are not connected yet to AWS IoT, wait 1 second and try again.
            if not isConnected:
                time.sleep(1)
                continue
            # Read a LoRa message, which contains the device state.  TODO: Check status.
            msg = lora_interface.readLoRaMessage()
            status = lora_interface.getLoRaStatus()

            # TODO: Comment this section.
            if len(msg) == 0:
                msg = '''{
                    "temperature": 27.3,
                    "humidity": 88
                }'''

            # If no message available, wait 1 second and try again.
            if len(msg) == 0:
                time.sleep(1)
                continue
            # Assume msg contains a JSON string with sensor names and values like
            # {
            #    "temperature": 27.3,
            #    "humidity": 88
            # }
            device_state = json.loads(msg)
            # Set the timestamp if not present.
            if device_state.get("timestamp") is None:
                device_state["timestamp"] = datetime.datetime.now().isoformat()

            # Send the reported device state to AWS IoT.
            payload = {
                "state": {
                    "reported": device_state
                }
            }
            print("Sending sensor data to AWS IoT...\n" +
                  json.dumps(payload, indent=4, separators=(',', ': ')))
            # Publish our sensor data to AWS IoT via the gateway topic and the LoRa topic.
            device_address = 2  # TODO
            device_topic = convert_lora_address_to_mqtt_topic(device_address)
            gateway_topic = convert_lora_address_to_mqtt_topic(gateway_address)
            client.publish(device_topic + "/shadow/update", json.dumps(payload))
            client.publish(gateway_topic + "/shadow/update", json.dumps(payload))
            print("Sent to AWS IoT")
            # Wait 30 seconds before sending the next message.
            time.sleep(30)

        except KeyboardInterrupt:
            # Stop the program when we press Ctrl-C.
            break
        except Exception as e:
            # For all other errors, we wait a while and resume.
            print("Exception: " + str(e))
            time.sleep(10)
            continue

# This is called when we are connected to AWS IoT via MQTT.
# We subscribe for notifications of desired state updates.
def on_connect(client, userdata, flags, rc):
    global isConnected
    isConnected = True
    print("Connected to AWS IoT")
    # Subscribe to MQTT topics for the gateway and all devices so that we will receive notifications of updates.
    gateway_topic = convert_lora_address_to_mqtt_topic(gateway_address)
    topic = gateway_topic + "/shadow/update/accepted"
    print("Subscribing to MQTT topic " + topic)
    client.subscribe(topic)
    for device_address in range(min_device_address, max_device_address):
        device_topic = convert_lora_address_to_mqtt_topic(device_address)
        topic = device_topic + "/shadow/update/accepted"
        print("Subscribing to MQTT topic " + topic)
        client.subscribe(topic)

# This is called when we receive a subscription notification from AWS IoT.
# If this is an actuation command, we execute it.
def on_message(client, userdata, msg):
    # Convert the JSON payload to a Python dictionary.
    # The payload is in binary format so we need to decode as UTF-8.
    payload2 = json.loads(msg.payload.decode("utf-8"))
    print("Received message, topic: " + msg.topic + ", payload:\n" +
          json.dumps(payload2, indent=4, separators=(',', ': ')))
    # If there is a desired state in this message, then we send to the LoRa device.
    if payload2.get("state") is not None and payload2["state"].get("desired") is not None:
        # Get the desired state and convert to JSON.
        desired_state = payload2["state"]["desired"]
        msg = json.dumps(desired_state)
        # Send to LoRa device.
        lora_address = convert_mqtt_topic_to_lora_address(msg.topic)
        status = lora_interface.sendLoRaMessage(lora_address, msg)  # TODO: Check status.

def convert_lora_address_to_mqtt_topic(address):
    # Address contains a number like 888.  Return topic $aws/things/l888.
    address = 2
    topic = "$aws/things/l{:0>3}".format(address)
    return topic

def convert_mqtt_topic_to_lora_address(topic):
    # Topic is of the format $aws/things/l888/... Return 888.
    topic_split = topic.split("/")
    address2 = topic_split[2]  # "l888"
    address3 = address2[1:]  # "888"
    address4 = int(address3)  # 888
    return address4

# Print out log messages for tracing.
def on_log(client, userdata, level, buf):
    print("Log: " + buf)

# Start the main program.
main()
<|endoftext|>


<commit_before>#!/usr/bin/env python

"""Tests pycits wrapper for muscle."""

import os
import shutil

from pycits import muscle
from pycits.tools import NotExecutableError

from nose.tools import nottest, assert_equal, with_setup

# INPUT DATA
INDIR = os.path.join("tests", "test_data")
OUTDIR = os.path.join("tests", "test_out", "muscle")
INFILE = os.path.join(INDIR, "dedup_test.fasta")
OUTFILE = os.path.join(OUTDIR, "dedup_test_aln.fasta")

# TARGET OUTPUT DATA
TARGET = os.path.join("tests", "test_targets", "muscle",
                      "muscle_tests.fasta")

# The setup_outdir() function decorates functions by creating the appropriate
# output directory tree
def setup_outdir():
    """Set up test fixtures"""
    try:
        shutil.rmtree(OUTDIR)
    except FileNotFoundError:
        pass
    os.makedirs(OUTDIR, exist_ok=True)

def test_muscle_path():
    """muscle executable is in $PATH"""
    muscle_aln = muscle.Muscle("muscle")

def test_muscle_exec_notexist():
    """error thrown if muscle executable does not exist"""
    try:
        bc = muscle.Muscle(os.path.join(".", "muscle"))
    except NotExecutableError:
        return True
    else:
        return False

def test_muscle_cmd():
    """muscle wrapper returns correct form of cmd-line"""
    muscle_aln = muscle.Muscle("muscle")
    muscle_result = muscle_aln.run(INFILE, OUTFILE, dry_run=True)
    target = ' '.join(['muscle',
                       '-in', INFILE,
                       '-out', OUTFILE])
    assert_equal(muscle_result.command, target)

@with_setup(setup_outdir)
def test_muscle_exec():
    """muscle aligns test data"""
    muscle_aln = muscle.Muscle("muscle")
    result = muscle_aln.run(INFILE, OUTFILE)
    with open(TARGET, "r") as target_fh:
        with open(result.outfile, "r") as test_fh:
            assert_equal(target_fh.read(), test_fh.read())
<commit_msg>restore test for muscle non-executable error<commit_after>#!/usr/bin/env python

"""Tests pycits wrapper for muscle."""

import os
import shutil

from pycits import muscle
from pycits.tools import NotExecutableError

from nose.tools import nottest, assert_equal, with_setup

# INPUT DATA
INDIR = os.path.join("tests", "test_data")
OUTDIR = os.path.join("tests", "test_out", "muscle")
INFILE = os.path.join(INDIR, "dedup_test.fasta")
OUTFILE = os.path.join(OUTDIR, "dedup_test_aln.fasta")

# TARGET OUTPUT DATA
TARGET = os.path.join("tests", "test_targets", "muscle",
                      "muscle_tests.fasta")

# The setup_outdir() function decorates functions by creating the appropriate
# output directory tree
def setup_outdir():
    """Set up test fixtures"""
    try:
        shutil.rmtree(OUTDIR)
    except FileNotFoundError:
        pass
    os.makedirs(OUTDIR, exist_ok=True)

def test_muscle_path():
    """muscle executable is in $PATH"""
    muscle_aln = muscle.Muscle("muscle")

def test_muscle_exec_notexist():
    """error thrown if muscle executable does not exist"""
    try:
        bc = muscle.Muscle(os.path.join(".", "muscle"))
    except NotExecutableError:
        return True
    else:
        return False

def test_muscle_cmd():
    """muscle wrapper returns correct form of cmd-line"""
    muscle_aln = muscle.Muscle("muscle")
    muscle_result = muscle_aln.run(INFILE, OUTFILE, dry_run=True)
    target = ' '.join(['muscle',
                       '-in', INFILE,
                       '-out', OUTFILE])
    assert_equal(muscle_result.command, target)

@with_setup(setup_outdir)
def test_muscle_exec():
    """muscle aligns test data"""
    muscle_aln = muscle.Muscle("muscle")
    result = muscle_aln.run(INFILE, OUTFILE)
    with open(TARGET, "r") as target_fh:
        with open(result.outfile, "r") as test_fh:
            assert_equal(target_fh.read(), test_fh.read())

def test_muscle_notexec():
    """error thrown if muscle exe not executable"""
    try:
        bc = muscle.Muscle("LICENSE")
    except NotExecutableError:
        return True
    else:
        return False
<|endoftext|>


<commit_before>from __future__ import absolute_import
from __future__ import print_function
from __future__ import unicode_literals

from sys import executable as python

def info(*msg):
    from sys import stdout
    stdout.write(' '.join(str(x) for x in msg))
    stdout.write('\n')
    stdout.flush()

def random_string():
    """return a short suffix that shouldn't collide with any subsequent calls"""
    import os
    import string
    import random

    return '{}.{}'.format(
        os.getpid(),
        ''.join(random.choice(string.ascii_lowercase) for x in range(8)),
    )

def sdist(setuppy, dst):
    import subprocess
    info('sdist', setuppy.dirname)
    subprocess.check_call(
        (python, 'setup.py', '--quiet', 'sdist', '--dist-dir', str(dst)),
        cwd=setuppy.dirname,
    )

def build_one(src, dst):
    setuppy = src.join('setup.py')
    if setuppy.exists():
        sdist(setuppy, dst)

        if src.join('wheelme').exists():
            wheel(src, dst)

        return True

def build_all(sources, dst):
    for source in sources:
        if build_one(source, dst):
            continue
        for source in source.listdir():
            if not source.check(dir=True):
                continue

            build_one(source, dst)

def flock(path):
    import os
    fd = os.open(path, os.O_CREAT)

    import fcntl
    fcntl.flock(fd, fcntl.LOCK_EX)  # exclusive

    return fd

class public_pypi_enabled(object):
    orig = None

    def __enter__(self):
        from os import environ
        self.orig = environ.pop('PIP_INDEX_URL', None)

    def __exit__(self, value, type_, traceback):
        from os import environ
        if self.orig is not None:
            environ['PIP_INDEX_URL'] = self.orig

def wheel(src, dst):
    import subprocess
    info('wheel', src)

    with public_pypi_enabled():
        subprocess.check_call(
            (python, '-m', 'pip.__main__', 'wheel', '--quiet', '--wheel-dir', str(dst), str(src)),
        )

def download_sdist(source, destination):
    import subprocess
    info('download sdist', source)
    with public_pypi_enabled():
        subprocess.check_call(
            (python, '-m', 'pip.__main__', 'install', '--quiet', '-d', str(destination), str(source), '--no-use-wheel', '--no-deps'),
        )

def make_sdists(sources, destination):
    destination.dirpath().ensure(dir=True)

    lock = destination.new(ext='lock')
    flock(lock.strpath)

    staging = destination.new(ext=random_string() + '.tmp')
    staging.ensure(dir=True)

    build_all(sources, staging)
    wheel('argparse', staging)
    wheel('coverage-enable-subprocess', staging)
    download_sdist('coverage', staging)
    download_sdist('coverage-enable-subprocess', staging)

    if destination.islink():
        old = destination.readlink()
    else:
        old = None

    link = staging.new(ext='ln')
    link.mksymlinkto(staging, absolute=False)
    link.rename(destination)

    if old is not None:
        destination.dirpath(old).remove()

def main():
    from sys import argv
    argv = argv[1:]
    sources, destination = argv[:-1], argv[-1]

    from py._path.local import LocalPath
    sources = tuple([
        LocalPath(src) for src in sources
    ])
    destination = LocalPath(destination)

    return make_sdists(sources, destination)

if __name__ == '__main__':
    exit(main())
<commit_msg>random-string: back to my old implementation<commit_after>from __future__ import absolute_import
from __future__ import print_function
from __future__ import unicode_literals

from sys import executable as python

def info(*msg):
    from sys import stdout
    stdout.write(' '.join(str(x) for x in msg))
    stdout.write('\n')
    stdout.flush()

def random_string():
    """return a short suffix that shouldn't collide with any subsequent calls"""
    import os
    import base64

    return '.'.join((
        str(os.getpid()),
        base64.urlsafe_b64encode(os.urandom(3)).decode('US-ASCII'),
    ))

def sdist(setuppy, dst):
    import subprocess
    info('sdist', setuppy.dirpath().basename)
    subprocess.check_call(
        (python, 'setup.py', '--quiet', 'sdist', '--dist-dir', str(dst)),
        cwd=setuppy.dirname,
    )

def build_one(src, dst):
    setuppy = src.join('setup.py')
    if setuppy.exists():
        sdist(setuppy, dst)

        if src.join('wheelme').exists():
            wheel(src, dst)

        return True

def build_all(sources, dst):
    for source in sources:
        if build_one(source, dst):
            continue
        for source in sorted(source.listdir()):
            if not source.check(dir=True):
                continue

            build_one(source, dst)

def flock(path):
    import os
    fd = os.open(path, os.O_CREAT)

    import fcntl
    fcntl.flock(fd, fcntl.LOCK_EX)  # exclusive

    return fd

class public_pypi_enabled(object):
    orig = None

    def __enter__(self):
        from os import environ
        self.orig = environ.pop('PIP_INDEX_URL', None)

    def __exit__(self, value, type_, traceback):
        from os import environ
        if self.orig is not None:
            environ['PIP_INDEX_URL'] = self.orig

def wheel(src, dst):
    import subprocess
    info('wheel', src)

    with public_pypi_enabled():
        subprocess.check_call(
            (python, '-m', 'pip.__main__', 'wheel', '--quiet', '--wheel-dir', str(dst), str(src)),
        )

def download_sdist(source, destination):
    import subprocess
    info('download sdist', source)
    with public_pypi_enabled():
        subprocess.check_call(
            (python, '-m', 'pip.__main__', 'install', '--quiet', '-d', str(destination), str(source), '--no-use-wheel', '--no-deps'),
        )

def make_sdists(sources, destination):
    destination.dirpath().ensure(dir=True)

    lock = destination.new(ext='lock')
    flock(lock.strpath)

    staging = destination.new(ext=random_string() + '.tmp')
    staging.ensure(dir=True)

    build_all(sources, staging)
    wheel('argparse', staging)
    wheel('coverage-enable-subprocess', staging)
    download_sdist('coverage', staging)
    download_sdist('coverage-enable-subprocess', staging)

    if destination.islink():
        old = destination.readlink()
    else:
        old = None

    link = staging.new(ext='ln')
    link.mksymlinkto(staging, absolute=False)
    link.rename(destination)

    if old is not None:
        destination.dirpath(old).remove()

def main():
    from sys import argv
    argv = argv[1:]
    sources, destination = argv[:-1], argv[-1]

    from py._path.local import LocalPath
    sources = tuple([
        LocalPath(src) for src in sources
    ])
    destination = LocalPath(destination)

    return make_sdists(sources, destination)

if __name__ == '__main__':
    exit(main())
<|endoftext|>


<commit_before>from django.db import transaction
from rdrf.models import Registry
from rdrf.models import RegistryForm
from rdrf.models import Section
from rdrf.models import CommonDataElement
from rdrf.utils import form_key
from rdrf.form_view import FormView
from django.test import RequestFactory

from registry.patients.models import Patient
import csv
from string import strip

class DataDictionaryFile:
    ROW_DELIMITER = '\t'
    FIELD_NUM_COLUMN = 1
    FORM_NAME_COLUMN = 6
    SECTION_NAME_COLUMN = 9
    CDE_NAME_COLUMN = 10

class DataImporterError(Exception):
    pass

class FormError(Exception):
    # represent a form validation error
    pass

class FieldType:
    CDE = 1
    DEMOGRAPHICS = 1
    PEDIGREE_FORM = 2
    CONSENT = 3

class RowWrapper(object):

    def _init__(self, registry_model, cde_field_num_map, row_dict):
        self.registry_model = registry_model
        self.cde_field_num_map = cde_field_num_map
        self.row_dict = row_dict

    def __getitem__(self, tuple_or_string):
        if type(tuple_or_string) is tuple:
            form_model, section_model, cde_model = tuple_or_string
            key = form_model.name + "/" + section_model.code + "/" + cde_model.code
            field_num = self.cde_field_num_map.get(key, None)
        else:
            field_num = self.cde_field_num_map.get(tuple_or_string, None)

        if field_num is None:
            raise DataImporterError("No fieldnum for %s" % tuple_or_string)

        try:
            value = self.row_dict[field_num]
        except KeyError:
            raise DataImporterError(
                "fieldnum %s not in row %s" % (field_num, self.row_dict))
        # return value suitable for db or mongo
        if form_model:
            # need to ensure display values are converted to codes
            transformed_value = self._get_transformed_value(
                form_model, section_model, cde_model, value)
            return transformed_value
        else:
            # demographics / consents etc
            return value

    def _get_transformed_value(self, form_model, section_model, cde_model, value):
        return value  # todo

class DataImporter(object):

    def __init__(self, registry_code, data_dictionary, import_file):
        self.registry_code = registry_code
        self.registry_model = Registry.objects.get(code=self.registry_code)
        self.data_dictionary = data_dictionary
        self.import_file = import_file
        self.created_patient_ids = []
        self.field_map = self._build_map()
        self.request_factory = RequestFactory()

    def _build_map(self):
        # map cde codes to field nums: field nums are unique ids of
        # form/section/cdes  or consents or demographic fields
        field_map = {}
        with open(self.data_dictionary) as dd:
            lines = map(strip, dd.readlines())[1:]  # skip header
            columns = lines.split(DataDictionaryFile.ROW_DELIMITER)
            for line in lines:
                field_num = columns[DataDictionaryFile.FIELD_NUM_COLUMN]
                spec = self._get_form_section_cde(columns)
                if type(spec) is tuple:
                    form_model, section_model, cde_model = spec
                    field_map[(FieldType.CDE, form_model, section_model, cde_model)] = field_num
                else:
                    # field is demographic or consent
                    form_name = columns[DataDictionaryFile.FORM_NAME_COLUMN]
                    section_name = columns[DataDictionaryFile.SECTION_NAME_COLUMN]
                    field_name = columns[DataDictionaryFile.CDE_NAME_COLUMN]
                    if form_name == "Demographics":
                        field_map[(FieldType.DEMOGRAPHICS, form_name, section_name, field_name)] = field_num
                    elif form_name == "Consent":
                        field_map[(FieldType.CONSENT, form_name, section_name, field_name)] = field_num
                    elif form_name == "Pedigree":
                        field_map[(FieldType.PEDIGREE_FORM, form_name, section_name, field_name)] = field_num

                    else:
                        raise DataImporterError("Unknown form: %s" % form_name)

        print field_map
        return field_map

    def _get_form_section_cde(self, columns):
        # return (form_model, section_model, cde_model)
        # if this makes sense otherwise None
        field_num = columns[DataDictionaryFile.FIELD_NUM_COLUMN]
        form_name = columns[DataDictionaryFile.FORM_NAME_COLUMN]
        section_name = columns[DataDictionaryFile.SECTION_NAME_COLUMN]
        cde_name = columns[DataDictionaryFile.CDE_NAME_COLUMN]

        try:
           form_model = RegistryForm.objects.get(name=form_name,
                                                 registry=self.registry_model)
        except Registry.DoesNotExist:
            return None

        # get the section in the form ...
        section_models = [section_model for section_model in form_model.section_models if section_model.display_name == section_name]
        if len(section_models) == 1:
            section_model = section_models[0]
        else:
            return None

        cde_models = [cde_model for cde_model in section_model.cde_models if cde_model.name == cde_name]
        if len(cde_models) == 1:
            cde_model = cde_models[0]
            return form_model, section_model, cde_model

    def run(self):
        try:
            with transaction.atomic():
                self._run()
        except Exception, ex:
            self._rollback_mongo()

    def _rollback_mongo(self):
        print "TODO"

    def _create_patient_model(self, row):
        patient = Patient()
        patient.family_name = row["family_name"]
        patient.given_names = row["given_names"]
        patient.date_of_birth = row["date_of_birth"]
        patient.sex = row["sex"]

    def _run(self):
        for row in self._get_rows():
            # row is RowWrapper
            patient_model = self._create_patient_model(row)
            patient_model.save()

            for form_model in self.registry_model.forms:
                form_data = self._get_form_data(form_model, row)
                try:
                    self.submit_form_data(patient_model, form_model, form_data)
                except FormError, fex:
                    print "Error submitting form"

    def _get_rows(self):
        with open(self.import_file, "rb") as csv_file:
            reader = csv.CSVReader(csv_file)
            for row_dict in reader:
                wrapped_row = RowWrapper(self.registry_model, self.field_map, row_dict)
                yield wrapped_row

    def _get_form_data(self, form_model, row):
        form_data = {}
        for section_model in form_model.section_models:
            if not section_model.allow_multiple:
                for cde_model in section_model.cde_models:
                    key = form_key(form_model, section_model, cde_model)
                    value = row[(form_model, section_model, cde_model)]
                form_data[key] = value
        return form_data

    def _submit_form_data(self, patient_model, form_model, form_data):
        request = self._create_request(form_model, form_data)
        view = FormView()
        view.request = request
        default_context_model = None
        try:
            # need to test for validation errors - how ?
            default_context_model = patient_model.default_context
            response = view.post(request, self.registry_model.code, form_model.pk, patient_model.pk, default_context_model.pk)
        except Exception:
            pass

    def _create_request(self, patient_model, form_model, form_data):
        url = "/%s/forms/%s/%s" % (form_model.registry.code, form_model.pk, patient_model.pk)
        request = self.request_factory.post(url, form_data)
        request.user = self.admin_user
        return request

<commit_msg>RDR-1273 Added admin user<commit_after>from django.db import transaction
from rdrf.models import Registry
from rdrf.models import RegistryForm
from rdrf.models import Section
from rdrf.models import CommonDataElement
from rdrf.utils import form_key
from rdrf.form_view import FormView
from django.test import RequestFactory

from registry.patients.models import Patient
import csv
from string import strip

class DataDictionaryFile:
    ROW_DELIMITER = '\t'
    FIELD_NUM_COLUMN = 1
    FORM_NAME_COLUMN = 6
    SECTION_NAME_COLUMN = 9
    CDE_NAME_COLUMN = 10

class DataImporterError(Exception):
    pass

class FormError(Exception):
    # represent a form validation error
    pass

class FieldType:
    CDE = 1
    DEMOGRAPHICS = 1
    PEDIGREE_FORM = 2
    CONSENT = 3

class RowWrapper(object):

    def _init__(self, registry_model, cde_field_num_map, row_dict):
        self.registry_model = registry_model
        self.cde_field_num_map = cde_field_num_map
        self.row_dict = row_dict

    def __getitem__(self, tuple_or_string):
        if type(tuple_or_string) is tuple:
            form_model, section_model, cde_model = tuple_or_string
            key = form_model.name + "/" + section_model.code + "/" + cde_model.code
            field_num = self.cde_field_num_map.get(key, None)
        else:
            field_num = self.cde_field_num_map.get(tuple_or_string, None)

        if field_num is None:
            raise DataImporterError("No fieldnum for %s" % tuple_or_string)

        try:
            value = self.row_dict[field_num]
        except KeyError:
            raise DataImporterError(
                "fieldnum %s not in row %s" % (field_num, self.row_dict))
        # return value suitable for db or mongo
        if form_model:
            # need to ensure display values are converted to codes
            transformed_value = self._get_transformed_value(
                form_model, section_model, cde_model, value)
            return transformed_value
        else:
            # demographics / consents etc
            return value

    def _get_transformed_value(self, form_model, section_model, cde_model, value):
        return value  # todo

class DataImporter(object):

    def __init__(self, registry_code, data_dictionary, import_file):
        self.registry_code = registry_code
        self.registry_model = Registry.objects.get(code=self.registry_code)
        self.data_dictionary = data_dictionary
        self.import_file = import_file
        self.created_patient_ids = []
        self.field_map = self._build_map()
        self.request_factory = RequestFactory()
        self.admin_user = CustomUser.objects.get(username="admin")

    def _build_map(self):
        # map cde codes to field nums: field nums are unique ids of
        # form/section/cdes  or consents or demographic fields
        field_map = {}
        with open(self.data_dictionary) as dd:
            lines = map(strip, dd.readlines())[1:]  # skip header
            columns = lines.split(DataDictionaryFile.ROW_DELIMITER)
            for line in lines:
                field_num = columns[DataDictionaryFile.FIELD_NUM_COLUMN]
                spec = self._get_form_section_cde(columns)
                if type(spec) is tuple:
                    form_model, section_model, cde_model = spec
                    field_map[(FieldType.CDE, form_model, section_model, cde_model)] = field_num
                else:
                    # field is demographic or consent
                    form_name = columns[DataDictionaryFile.FORM_NAME_COLUMN]
                    section_name = columns[DataDictionaryFile.SECTION_NAME_COLUMN]
                    field_name = columns[DataDictionaryFile.CDE_NAME_COLUMN]
                    if form_name == "Demographics":
                        field_map[(FieldType.DEMOGRAPHICS, form_name, section_name, field_name)] = field_num
                    elif form_name == "Consent":
                        field_map[(FieldType.CONSENT, form_name, section_name, field_name)] = field_num
                    elif form_name == "Pedigree":
                        field_map[(FieldType.PEDIGREE_FORM, form_name, section_name, field_name)] = field_num

                    else:
                        raise DataImporterError("Unknown form: %s" % form_name)

        print field_map
        return field_map

    def _get_form_section_cde(self, columns):
        # return (form_model, section_model, cde_model)
        # if this makes sense otherwise None
        field_num = columns[DataDictionaryFile.FIELD_NUM_COLUMN]
        form_name = columns[DataDictionaryFile.FORM_NAME_COLUMN]
        section_name = columns[DataDictionaryFile.SECTION_NAME_COLUMN]
        cde_name = columns[DataDictionaryFile.CDE_NAME_COLUMN]

        try:
           form_model = RegistryForm.objects.get(name=form_name,
                                                 registry=self.registry_model)
        except Registry.DoesNotExist:
            return None

        # get the section in the form ...
        section_models = [section_model for section_model in form_model.section_models if section_model.display_name == section_name]
        if len(section_models) == 1:
            section_model = section_models[0]
        else:
            return None

        cde_models = [cde_model for cde_model in section_model.cde_models if cde_model.name == cde_name]
        if len(cde_models) == 1:
            cde_model = cde_models[0]
            return form_model, section_model, cde_model

    def run(self):
        try:
            with transaction.atomic():
                self._run()
        except Exception, ex:
            self._rollback_mongo()

    def _rollback_mongo(self):
        print "TODO"

    def _create_patient_model(self, row):
        patient = Patient()
        patient.family_name = row["family_name"]
        patient.given_names = row["given_names"]
        patient.date_of_birth = row["date_of_birth"]
        patient.sex = row["sex"]

    def _run(self):
        for row in self._get_rows():
            # row is RowWrapper
            patient_model = self._create_patient_model(row)
            patient_model.save()

            for form_model in self.registry_model.forms:
                form_data = self._get_form_data(form_model, row)
                try:
                    self.submit_form_data(patient_model, form_model, form_data)
                except FormError, fex:
                    print "Error submitting form"

    def _get_rows(self):
        with open(self.import_file, "rb") as csv_file:
            reader = csv.CSVReader(csv_file)
            for row_dict in reader:
                wrapped_row = RowWrapper(self.registry_model, self.field_map, row_dict)
                yield wrapped_row

    def _get_form_data(self, form_model, row):
        form_data = {}
        for section_model in form_model.section_models:
            if not section_model.allow_multiple:
                for cde_model in section_model.cde_models:
                    key = form_key(form_model, section_model, cde_model)
                    value = row[(form_model, section_model, cde_model)]
                form_data[key] = value
        return form_data

    def _submit_form_data(self, patient_model, form_model, form_data):
        request = self._create_request(patient_model, form_model, form_data)
        view = FormView()
        view.request = request
        default_context_model = None
        try:
            # need to test for validation errors - how ?
            default_context_model = patient_model.default_context
            response = view.post(request, self.registry_model.code, form_model.pk, patient_model.pk, default_context_model.pk)
            validation_errors = self._get_validation_errors(response)

        except Exception:
            pass

    def _create_request(self, patient_model, form_model, form_data):
        url = "/%s/forms/%s/%s" % (form_model.registry.code, form_model.pk, patient_model.pk)
        request = self.request_factory.post(url, form_data)
        request.user = self.admin_user
        return request

    def _get_validation_errors(self, response):
        return [] #todo

<|endoftext|>


<commit_before>from pytest import mark
import six
import sqlalchemy as sa
from sqlalchemy_utils.types import timezone
from tests import TestCase

import dateutil
import pytz

class TestTimezoneType(TestCase):
    def create_models(self):
        class Visitor(self.Base):
            __tablename__ = 'document'
            id = sa.Column(sa.Integer, primary_key=True)
            timezone_dateutil = sa.Column(timezone.TimezoneType(
                                          backend='dateutil'))
            timezone_pytz = sa.Column(timezone.TimezoneType(backend='pytz'))

            def __repr__(self):
                return 'Visitor(%r)' % self.id

        self.Visitor = Visitor

    def test_parameter_processing(self):
        visitor = self.Visitor(
            timezone_dateutil=u'America/Los_Angeles',
            timezone_pytz=u'America/Los_Angeles'
        )

        self.session.add(visitor)
        self.session.commit()

        visitor_dateutil = self.session.query(self.Visitor).filter_by(
            timezone_dateutil='America/Los_Angeles').first()
        visitor_pytz = self.session.query(self.Visitor).filter_by(
            timezone_pytz='America/Los_Angeles').first()

        assert visitor_dateutil is not None
        assert visitor_pytz is not None
<commit_msg>Fix indentation<commit_after>from pytest import mark
import six
import sqlalchemy as sa
from sqlalchemy_utils.types import timezone
from tests import TestCase

import dateutil
import pytz

class TestTimezoneType(TestCase):
    def create_models(self):
        class Visitor(self.Base):
            __tablename__ = 'visitor'
            id = sa.Column(sa.Integer, primary_key=True)
            timezone_dateutil = sa.Column(
                timezone.TimezoneType(backend='dateutil')
            )
            timezone_pytz = sa.Column(
                timezone.TimezoneType(backend='pytz')
            )

            def __repr__(self):
                return 'Visitor(%r)' % self.id

        self.Visitor = Visitor

    def test_parameter_processing(self):
        visitor = self.Visitor(
            timezone_dateutil=u'America/Los_Angeles',
            timezone_pytz=u'America/Los_Angeles'
        )

        self.session.add(visitor)
        self.session.commit()

        visitor_dateutil = self.session.query(self.Visitor).filter_by(
            timezone_dateutil=u'America/Los_Angeles'
        ).first()
        visitor_pytz = self.session.query(self.Visitor).filter_by(
            timezone_pytz=u'America/Los_Angeles'
        ).first()

        assert visitor_dateutil is not None
        assert visitor_pytz is not None
<|endoftext|>