<commit_before># -*- coding: utf-8 -*-
"""
.. wikisection:: overview
    :title: Plots

    The :mod:`biseqt.plots` module provides plotting utilities based on
    ``matplotlib`` functionality in addition to basic tools for inspecting
    binary classifiers.

    >>> from biseqt.plots import figure, save, plot_density, plot_roc, plot_ppv
    >>> X = ... # a list of numbers (the positive data set)
    >>> Y = ... # a list of numbers (the negatvie data set)
    >>> fig = biseqt.plots.figure()
    >>> plot_density(fig.subplot(4, 1, 1), X)
    >>> plot_density(fig.subplot(4, 1, 2), Y)
    >>> plot_roc(fig.subplot(4, 1, 3), X, Y)
    >>> plot_ppv(fig.subplot(4, 1, 4), X, Y)
    >>> save(fig, '/path/to/fig.png')
"""

import numpy as np
import scipy.stats
import matplotlib
from matplotlib.figure import Figure
from matplotlib.backends.backend_agg import FigureCanvas

matplotlib.rc('font', style='normal', size=8)

_default_scatter_kwargs = {
    's': 1,
    'color': 'k',
}

def _scatter(ax, *args, **kwargs):
    _kwargs = _default_scatter_kwargs.copy()
    _kwargs.update(kwargs)
    ax.scatter(*args, **_kwargs)

def figure():
    """Initializes new figure with the AGG backend_ and adds a canvas to it.

    .. _backend: http://matplotlib.org/faq/usage_faq.html#what-is-a-backend

    Returns:
        matplotlib.figure.Figure
    """
    fig = Figure()
    FigureCanvas(fig)  # add a canvas to figure
    return fig

def save(fig, path, **kwargs):
    """Saves figure to path. All keyword arguments are passed as is to
    ``matplotlib.figure.Figure.savefig``.

    Args:
        fig (matplotlib.figure.Figure)
        path (str)
    """
    for ax in fig.get_axes():
        ax.grid(True)
        ax.legend()
    fig.tight_layout()
    fig.savefig(path, **kwargs)

def plot_density(ax, sample, num_points=1000, **kwargs):
    """Plots the Guassian density estimate of the given sample. All un-named
    keyword arguments are passed as is to ``matplotlib.axes.Axes.plot``.

    Args:
        ax (matplotlib.axes.Axes): The matplotlib axes object to use.
        sample (list): A list of numbers; the sample set.

    Keyword Args:
        num_points (int): The sample size of the density estimate.
    """
    density = scipy.stats.gaussian_kde(sample)
    xs = np.linspace(min(sample), max(sample), num=num_points)
    ax.plot(xs, density(xs), **kwargs)

def cdf(sample, num_points=1000, value_range=None):
    """Calculate the cumulative distribution function of a given sample set
    based on its estimated density.

    Args:
        sample (list): A list of numbers; the sample set.

    Keyword Args:
        num_points (int): The sample size of the CDF.
        value_range (tuple): The minimum and maximum values where the CDF
            should be calculated; default is None in which case the bounds of
            the sample set are used.

    Returns:
        tuple:
            A tuple containing two lists: the values at which the CDF is
            reported and CDF values themselves.

    Note:
        The density estimation is done using ``scipy.stats.guassian_kde`` which
        requires a "reasonably large" sample set to behave regularly. For
        instance, if the sample size is too small (e.g. 3 data points only)
        the calculated CDF may not necessarily approach 1 at its higher end.
    """
    density = scipy.stats.gaussian_kde(sample)
    if value_range is None:
        value_range = min(sample), max(sample)
    xs = np.linspace(*value_range, num=num_points)
    cdf = scipy.integrate.cumtrapz(density(xs), xs)
    return xs, np.insert(cdf, [0], 0)

def plot_cdf(ax, sample, num_points=1000, **kwargs):
    """Plots the cumulative distribution of a given sample set. All unnamed
    keyword arguments are passed as is to ``matplotlib.axes.Axes.scatter``.

    Args:
        ax (matplotlib.axes.Axes): The matplotlib axes object to use.
        sample (list): A list of numbers; the sample set.

    Keyword Args:
        num_points (int): The sample size of the CDF.
    """
    xs, F = cdf(sample, num_points=num_points)
    ax.set_title('Cumulative Distribution')
    _scatter(ax, xs, F, **kwargs)

def roc(X, Y, classifier='>', num_points=1000):
    """Sample the ROC curve for a pair of sample sets labeled as *positive* and
    *negative*. The ROC curve is defined by the
    :math:`(\mathrm{FPR},\mathrm{TPR})` pairs of false and true positive rates
    obtained for various values of threshold. The rates are defined to be:

    .. math::
        \mathrm{FPR} = \\frac{f.p.}{f.p. + t.n.}

        \mathrm{TPR} = \\frac{t.p.}{t.p. + f.n.}

    where :math:`t.p., f.p., t.n., f.n.` stand for true and false positives and
    negatives.  Letting :math:`F_X,F_Y` be the cumulative distributions of the
    sample sets we get:

    .. math::
        \mathrm{FPR} = 1 - F_Y, \\text{ and } \mathrm{TPR} = 1 - F_X

    when the direction for positive selection is ``>``, and:

    .. math::
        \mathrm{FPR} = F_Y, \\text{ and } \mathrm{TPR} = F_X

    when the direction for positive selection is ``<``.

    Args:
        X (list): The "positive" sample set.
        Y (list): The "negative" sample set.

    Keyword Args:
        classifier (str): Either of ``>`` or ``<`` denoting the rule for
            positive classification: the former indicates positive
            classification if observed value is *larger* than threshold and
            the latter indicates positive classification if observed value is
            *smaller* than threshold; default is ``>``, i.e positively
            classified values are thought to be mostly larger than negatively
            classified values.
        num_points (int): The sample size of the ROC curve; translates to
            sample points of the CDF of each sample set.

    Returns:
        tuple:
            A tuple containing three lists: the parameter values (i.e threshold
            values) for which the ROC curve is sampled, the *false positive*
            rates and the *true positive* rates at every sample point.
    """
    assert classifier in '><'
    _all = list(X) + list(Y)
    value_range = min(_all), max(_all)
    xs, F_X = cdf(X, num_points=num_points, value_range=value_range)
    _, F_Y = cdf(Y, num_points=num_points, value_range=value_range)
    return (xs, 1 - F_Y, 1 - F_X) if classifier == '>' else (xs, F_Y, F_X)

def plot_roc(ax, X, Y, **kwargs):
    """Plots the ROC curve of two sample sets, cf. :func:`roc`. All keyword
    arguments are passed as is to ``matplotlib.axes.Axes.scatter``.

    Args:
        ax (matplotlib.axes.Axes): The matplotlib axes object to use.
        X (list): The "positive" sample set (horizontal axis).
        Y (list): The "negative" sample set (vertical axis).
    """
    ax.set_xlim([-.1, 1.1])
    ax.set_ylim([-.1, 1.1])
    ax.set_xlabel('False positive rate')
    ax.set_ylabel('True positive rate')
    _, x, y = roc(X, Y)
    _scatter(ax, x, y, **kwargs)

def ppv(X, Y, classifier='>', num_points=1000):
    """Samples the *positive predictive value* for a pair of sample sets
    labeled as *positive* and *negative*. The positive negative predictive
    value is defined as:

    .. math::
        \mathrm{PPV} = \\frac{t.p.}{t.p. + f.p.}

    where :math:`t.p., f.p., t.n., f.n.` stand for true and false positives and
    negatives. Depending on the classifier direction this translates to:

    .. math::
        \mathrm{PPV} = \\frac{(1-F_X)|X|}{(1-F_X)|X| + (1-F_Y)|Y|}

    when the direction for positive selection is ``>``, and:

    .. math::
        \mathrm{PPV} = \\frac{F_X|Y|}{F_X|X| + F_Y|Y|}

    when the direction for positive selection is ``<``.

    Args:
        X (list): The "positive" sample set.
        Y (list): The "negative" sample set.

    Keyword Args:
        classifier (str): Either of ``>`` or ``<`` denoting the rule for
            positive classification: the former indicates positive
            classification if observed value is *larger* than threshold and
            the latter indicates positive classification if observed value is
            *smaller* than threshold; default is ``>``, i.e positively
            classified values are thought to be mostly larger than negatively
            classified values.
        num_points (int): The number of sample points.

    Returns:
        tuple:
            A tuple containing two lists: the parameter values where the PPV is
            sampled, and PPV at each point.
    """
    assert classifier in '><'
    _all = list(X) + list(Y)
    value_range = min(_all), max(_all)
    params, F_X = cdf(X, num_points=num_points, value_range=value_range)
    _, F_Y = cdf(Y, num_points=num_points, value_range=value_range)

    if classifier == '>':
        Nc_X, Nc_Y = (1 - F_X) * len(X), (1 - F_Y) * len(Y)
        with np.errstate(divide='ignore', invalid='ignore'):
            # don't complain about points where denominator is 0
            return params, Nc_X / (Nc_X + Nc_Y)
    else:
        N_X, N_Y = F_X * len(X), F_Y * len(Y)
        with np.errstate(divide='ignore', invalid='ignore'):
            return params, N_X / (N_X + N_Y)

def npv(X, Y, classifier='>', num_points=1000):
    """Samples the *negative predictive value* for a pair of sample sets
    labeled as *positive* and *negative*. The positive negative predictive
    value is defined as:

    .. math::
        \mathrm{NPV} = \\frac{t.n.}{t.n. + f.n.}

    where :math:`t.p., f.p., t.n., f.n.` stand for true and false positives and
    negatives. Depending on the classifier direction this translates to:

    .. math::
        \mathrm{NPV} = \\frac{F_Y|Y|}{F_X|X| + F_Y|Y|}

    when the direction for positive selection is ``>``, and:

    .. math::
        \mathrm{NPV} = \\frac{(1-F_Y)|Y|}{(1-F_X)|X| + (1-F_Y)|Y|}

    when the direction for positive selection is ``<``.

    Args:
        X (list): The "positive" sample set.
        Y (list): The "negative" sample set.

    Keyword Args:
        classifier (str): Either of ``>`` or ``<``, cf. :func:`ppv`.
        num_points (int): The number of sample points.

    Returns:
        tuple:
            A tuple containing two lists: the parameter values where the PPV is
            sampled, and NPV at each point.
    """
    assert classifier in '><'
    _all = list(X) + list(Y)
    value_range = min(_all), max(_all)
    params, F_X = cdf(X, num_points=num_points, value_range=value_range)
    _, F_Y = cdf(Y, num_points=num_points, value_range=value_range)

    if classifier == '>':
        N_X, N_Y = F_X * len(X), F_Y * len(Y)
        with np.errstate(divide='ignore', invalid='ignore'):
            return params, N_Y / (N_X + N_Y)
    else:
        Nc_X, Nc_Y = (1 - F_X) * len(X), (1 - F_Y) * len(Y)
        with np.errstate(divide='ignore', invalid='ignore'):
            return params, Nc_Y / (Nc_X + Nc_X)

def plot_ppv(ax, X, Y, classifier='>', num_points=1000, **kwargs):
    """Plots the positive predictive value over a range of parameters. All
    keyword arguments are passed as is to ``matplotlib.axes.Axes.scatter``.

    Args:
        X (list): The "positive" sample set.
        Y (list): The "negative" sample set.

    Keyword Args:
        classifier (str): Either of ``>`` or ``<``, cf. :func:`ppv`.
        num_points (int): The number of sample points.
    """
    params, ppvs = ppv(X, Y, classifier=classifier, num_points=num_points)
    ax.set_title('Positive Predictive Value')
    _scatter(ax, params, ppvs, **kwargs)

def plot_npv(ax, X, Y, classifier='>', num_points=1000, **kwargs):
    """Plots the negative predictive value over a range of parameters. All
    keyword arguments are passed as is to ``matplotlib.axes.Axes.scatter``.

    Args:
        X (list): The "positive" sample set.
        Y (list): The "negative" sample set.

    Keyword Args:
        classifier (str): Either of ``>`` or ``<``, cf. :func:`ppv`.
        num_points (int): The number of sample points.
    """
    params, npvs = npv(X, Y, classifier=classifier, num_points=num_points)
    ax.set_title('Negative Predictive Value')
    _scatter(ax, params, npvs, **kwargs)
<commit_msg>save figures with dpi=500 by default<commit_after># -*- coding: utf-8 -*-
"""
.. wikisection:: overview
    :title: Plots

    The :mod:`biseqt.plots` module provides plotting utilities based on
    ``matplotlib`` functionality in addition to basic tools for inspecting
    binary classifiers.

    >>> from biseqt.plots import figure, save, plot_density, plot_roc, plot_ppv
    >>> X = ... # a list of numbers (the positive data set)
    >>> Y = ... # a list of numbers (the negatvie data set)
    >>> fig = biseqt.plots.figure()
    >>> plot_density(fig.subplot(4, 1, 1), X)
    >>> plot_density(fig.subplot(4, 1, 2), Y)
    >>> plot_roc(fig.subplot(4, 1, 3), X, Y)
    >>> plot_ppv(fig.subplot(4, 1, 4), X, Y)
    >>> save(fig, '/path/to/fig.png')
"""

import numpy as np
import scipy.stats
import matplotlib
from matplotlib.figure import Figure
from matplotlib.backends.backend_agg import FigureCanvas

matplotlib.rc('font', style='normal', size=8)

_default_scatter_kwargs = {
    's': 1,
    'color': 'k',
}
_default_savefig_kwargs = {
    'dpi': 500,
}

def _scatter(ax, *args, **kwargs):
    _kwargs = _default_scatter_kwargs.copy()
    _kwargs.update(kwargs)
    ax.scatter(*args, **_kwargs)

def figure():
    """Initializes new figure with the AGG backend_ and adds a canvas to it.

    .. _backend: http://matplotlib.org/faq/usage_faq.html#what-is-a-backend

    Returns:
        matplotlib.figure.Figure
    """
    fig = Figure()
    FigureCanvas(fig)  # add a canvas to figure
    return fig

def save(fig, path, **kwargs):
    """Saves figure to path. All keyword arguments are passed as is to
    ``matplotlib.figure.Figure.savefig``.

    Args:
        fig (matplotlib.figure.Figure)
        path (str)
    """
    for ax in fig.get_axes():
        ax.grid(True)
        ax.legend()
    fig.tight_layout()
    _kwargs = _default_savefig_kwargs
    _kwargs.update(kwargs)
    fig.savefig(path, **_kwargs)

def plot_density(ax, sample, num_points=1000, **kwargs):
    """Plots the Guassian density estimate of the given sample. All un-named
    keyword arguments are passed as is to ``matplotlib.axes.Axes.plot``.

    Args:
        ax (matplotlib.axes.Axes): The matplotlib axes object to use.
        sample (list): A list of numbers; the sample set.

    Keyword Args:
        num_points (int): The sample size of the density estimate.
    """
    density = scipy.stats.gaussian_kde(sample)
    xs = np.linspace(min(sample), max(sample), num=num_points)
    ax.plot(xs, density(xs), **kwargs)

def cdf(sample, num_points=1000, value_range=None):
    """Calculate the cumulative distribution function of a given sample set
    based on its estimated density.

    Args:
        sample (list): A list of numbers; the sample set.

    Keyword Args:
        num_points (int): The sample size of the CDF.
        value_range (tuple): The minimum and maximum values where the CDF
            should be calculated; default is None in which case the bounds of
            the sample set are used.

    Returns:
        tuple:
            A tuple containing two lists: the values at which the CDF is
            reported and CDF values themselves.

    Note:
        The density estimation is done using ``scipy.stats.guassian_kde`` which
        requires a "reasonably large" sample set to behave regularly. For
        instance, if the sample size is too small (e.g. 3 data points only)
        the calculated CDF may not necessarily approach 1 at its higher end.
    """
    density = scipy.stats.gaussian_kde(sample)
    if value_range is None:
        value_range = min(sample), max(sample)
    xs = np.linspace(*value_range, num=num_points)
    cdf = scipy.integrate.cumtrapz(density(xs), xs)
    return xs, np.insert(cdf, [0], 0)

def plot_cdf(ax, sample, num_points=1000, **kwargs):
    """Plots the cumulative distribution of a given sample set. All unnamed
    keyword arguments are passed as is to ``matplotlib.axes.Axes.scatter``.

    Args:
        ax (matplotlib.axes.Axes): The matplotlib axes object to use.
        sample (list): A list of numbers; the sample set.

    Keyword Args:
        num_points (int): The sample size of the CDF.
    """
    xs, F = cdf(sample, num_points=num_points)
    ax.set_title('Cumulative Distribution')
    _scatter(ax, xs, F, **kwargs)

def roc(X, Y, classifier='>', num_points=1000):
    """Sample the ROC curve for a pair of sample sets labeled as *positive* and
    *negative*. The ROC curve is defined by the
    :math:`(\mathrm{FPR},\mathrm{TPR})` pairs of false and true positive rates
    obtained for various values of threshold. The rates are defined to be:

    .. math::
        \mathrm{FPR} = \\frac{f.p.}{f.p. + t.n.}

        \mathrm{TPR} = \\frac{t.p.}{t.p. + f.n.}

    where :math:`t.p., f.p., t.n., f.n.` stand for true and false positives and
    negatives.  Letting :math:`F_X,F_Y` be the cumulative distributions of the
    sample sets we get:

    .. math::
        \mathrm{FPR} = 1 - F_Y, \\text{ and } \mathrm{TPR} = 1 - F_X

    when the direction for positive selection is ``>``, and:

    .. math::
        \mathrm{FPR} = F_Y, \\text{ and } \mathrm{TPR} = F_X

    when the direction for positive selection is ``<``.

    Args:
        X (list): The "positive" sample set.
        Y (list): The "negative" sample set.

    Keyword Args:
        classifier (str): Either of ``>`` or ``<`` denoting the rule for
            positive classification: the former indicates positive
            classification if observed value is *larger* than threshold and
            the latter indicates positive classification if observed value is
            *smaller* than threshold; default is ``>``, i.e positively
            classified values are thought to be mostly larger than negatively
            classified values.
        num_points (int): The sample size of the ROC curve; translates to
            sample points of the CDF of each sample set.

    Returns:
        tuple:
            A tuple containing three lists: the parameter values (i.e threshold
            values) for which the ROC curve is sampled, the *false positive*
            rates and the *true positive* rates at every sample point.
    """
    assert classifier in '><'
    _all = list(X) + list(Y)
    value_range = min(_all), max(_all)
    xs, F_X = cdf(X, num_points=num_points, value_range=value_range)
    _, F_Y = cdf(Y, num_points=num_points, value_range=value_range)
    return (xs, 1 - F_Y, 1 - F_X) if classifier == '>' else (xs, F_Y, F_X)

def plot_roc(ax, X, Y, **kwargs):
    """Plots the ROC curve of two sample sets, cf. :func:`roc`. All keyword
    arguments are passed as is to ``matplotlib.axes.Axes.scatter``.

    Args:
        ax (matplotlib.axes.Axes): The matplotlib axes object to use.
        X (list): The "positive" sample set (horizontal axis).
        Y (list): The "negative" sample set (vertical axis).
    """
    ax.set_xlim([-.1, 1.1])
    ax.set_ylim([-.1, 1.1])
    ax.set_xlabel('False positive rate')
    ax.set_ylabel('True positive rate')
    _, x, y = roc(X, Y)
    _scatter(ax, x, y, **kwargs)

def ppv(X, Y, classifier='>', num_points=1000):
    """Samples the *positive predictive value* for a pair of sample sets
    labeled as *positive* and *negative*. The positive negative predictive
    value is defined as:

    .. math::
        \mathrm{PPV} = \\frac{t.p.}{t.p. + f.p.}

    where :math:`t.p., f.p., t.n., f.n.` stand for true and false positives and
    negatives. Depending on the classifier direction this translates to:

    .. math::
        \mathrm{PPV} = \\frac{(1-F_X)|X|}{(1-F_X)|X| + (1-F_Y)|Y|}

    when the direction for positive selection is ``>``, and:

    .. math::
        \mathrm{PPV} = \\frac{F_X|Y|}{F_X|X| + F_Y|Y|}

    when the direction for positive selection is ``<``.

    Args:
        X (list): The "positive" sample set.
        Y (list): The "negative" sample set.

    Keyword Args:
        classifier (str): Either of ``>`` or ``<`` denoting the rule for
            positive classification: the former indicates positive
            classification if observed value is *larger* than threshold and
            the latter indicates positive classification if observed value is
            *smaller* than threshold; default is ``>``, i.e positively
            classified values are thought to be mostly larger than negatively
            classified values.
        num_points (int): The number of sample points.

    Returns:
        tuple:
            A tuple containing two lists: the parameter values where the PPV is
            sampled, and PPV at each point.
    """
    assert classifier in '><'
    _all = list(X) + list(Y)
    value_range = min(_all), max(_all)
    params, F_X = cdf(X, num_points=num_points, value_range=value_range)
    _, F_Y = cdf(Y, num_points=num_points, value_range=value_range)

    if classifier == '>':
        Nc_X, Nc_Y = (1 - F_X) * len(X), (1 - F_Y) * len(Y)
        with np.errstate(divide='ignore', invalid='ignore'):
            # don't complain about points where denominator is 0
            return params, Nc_X / (Nc_X + Nc_Y)
    else:
        N_X, N_Y = F_X * len(X), F_Y * len(Y)
        with np.errstate(divide='ignore', invalid='ignore'):
            return params, N_X / (N_X + N_Y)

def npv(X, Y, classifier='>', num_points=1000):
    """Samples the *negative predictive value* for a pair of sample sets
    labeled as *positive* and *negative*. The positive negative predictive
    value is defined as:

    .. math::
        \mathrm{NPV} = \\frac{t.n.}{t.n. + f.n.}

    where :math:`t.p., f.p., t.n., f.n.` stand for true and false positives and
    negatives. Depending on the classifier direction this translates to:

    .. math::
        \mathrm{NPV} = \\frac{F_Y|Y|}{F_X|X| + F_Y|Y|}

    when the direction for positive selection is ``>``, and:

    .. math::
        \mathrm{NPV} = \\frac{(1-F_Y)|Y|}{(1-F_X)|X| + (1-F_Y)|Y|}

    when the direction for positive selection is ``<``.

    Args:
        X (list): The "positive" sample set.
        Y (list): The "negative" sample set.

    Keyword Args:
        classifier (str): Either of ``>`` or ``<``, cf. :func:`ppv`.
        num_points (int): The number of sample points.

    Returns:
        tuple:
            A tuple containing two lists: the parameter values where the PPV is
            sampled, and NPV at each point.
    """
    assert classifier in '><'
    _all = list(X) + list(Y)
    value_range = min(_all), max(_all)
    params, F_X = cdf(X, num_points=num_points, value_range=value_range)
    _, F_Y = cdf(Y, num_points=num_points, value_range=value_range)

    if classifier == '>':
        N_X, N_Y = F_X * len(X), F_Y * len(Y)
        with np.errstate(divide='ignore', invalid='ignore'):
            return params, N_Y / (N_X + N_Y)
    else:
        Nc_X, Nc_Y = (1 - F_X) * len(X), (1 - F_Y) * len(Y)
        with np.errstate(divide='ignore', invalid='ignore'):
            return params, Nc_Y / (Nc_X + Nc_X)

def plot_ppv(ax, X, Y, classifier='>', num_points=1000, **kwargs):
    """Plots the positive predictive value over a range of parameters. All
    keyword arguments are passed as is to ``matplotlib.axes.Axes.scatter``.

    Args:
        X (list): The "positive" sample set.
        Y (list): The "negative" sample set.

    Keyword Args:
        classifier (str): Either of ``>`` or ``<``, cf. :func:`ppv`.
        num_points (int): The number of sample points.
    """
    params, ppvs = ppv(X, Y, classifier=classifier, num_points=num_points)
    ax.set_title('Positive Predictive Value')
    _scatter(ax, params, ppvs, **kwargs)

def plot_npv(ax, X, Y, classifier='>', num_points=1000, **kwargs):
    """Plots the negative predictive value over a range of parameters. All
    keyword arguments are passed as is to ``matplotlib.axes.Axes.scatter``.

    Args:
        X (list): The "positive" sample set.
        Y (list): The "negative" sample set.

    Keyword Args:
        classifier (str): Either of ``>`` or ``<``, cf. :func:`ppv`.
        num_points (int): The number of sample points.
    """
    params, npvs = npv(X, Y, classifier=classifier, num_points=num_points)
    ax.set_title('Negative Predictive Value')
    _scatter(ax, params, npvs, **kwargs)
<|endoftext|>


<commit_before>import tensorflow as tf
from tensorflow.python.ops import rnn_cell_impl

import bit_utils

class BitGRUCell(tf.nn.rnn_cell.GRUCell):

    def __init__(self,
                 num_units,
                 w_bit,
                 f_bit,
                 activation=None,
                 reuse=None,
                 kernel_initializer=None,
                 bias_initializer=None):
        super(BitGRUCell, self).__init__(num_units, activation,
                                         reuse, kernel_initializer, bias_initializer)
        self._w_bit = w_bit
        self._f_bit = f_bit

    def build(self, inputs_shape):
        def replace_w(x):
            if x.op.name.endswith('kernel'):
                return bit_utils.quantize_w(tf.tanh(x), bit=self._w_bit)
            else:
                return x
        with bit_utils.replace_variable(replace_w):
            super(BitGRUCell, self).build(inputs_shape)

    def call(self, inputs, state):
        if self._gate_linear is None:
            bias_ones = self._bias_initializer
            if self._bias_initializer is None:
                bias_ones = tf.constant_initializer(
                    1.0, dtype=inputs.dtype)
            with tf.variable_scope("gates"):  # Reset gate and update gate.
                self._gate_linear = rnn_cell_impl._Linear(
                    [inputs, state],
                    2 * self._num_units,
                    True,
                    bias_initializer=bias_ones,
                    kernel_initializer=self._kernel_initializer)

        value = tf.sigmoid(self._gate_linear([inputs, state]))
        r, u = tf.split(value=value, num_or_size_splits=2, axis=1)

        r_state = bit_utils.round_bit(r * state, bit=self._f_bit)
        if self._candidate_linear is None:
            with tf.variable_scope("candidate"):
                self._candidate_linear = rnn_cell_impl._Linear(
                    [inputs, r_state],
                    self._num_units,
                    True,
                    bias_initializer=self._bias_initializer,
                    kernel_initializer=self._kernel_initializer)
        c = self._activation(self._candidate_linear([inputs, r_state]))
        c = bit_utils.round_bit(c, bit=self._f_bit)
        new_h = bit_utils.round_bit(u * state + (1 - u) * c, bit=self._f_bit)
        return new_h, new_h

class BitLSTMCell(tf.nn.rnn_cell.BasicLSTMCell):
    pass
<commit_msg>bug fix: not quantizing weight<commit_after>import tensorflow as tf
from tensorflow.python.ops import rnn_cell_impl

import bit_utils

class BitGRUCell(tf.nn.rnn_cell.GRUCell):

    def __init__(self,
                 num_units,
                 w_bit,
                 f_bit,
                 activation=None,
                 reuse=None,
                 kernel_initializer=None,
                 bias_initializer=None):
        super(BitGRUCell, self).__init__(num_units, activation,
                                         reuse, kernel_initializer, bias_initializer)
        self._w_bit = w_bit
        self._f_bit = f_bit

    def call(self, inputs, state):
        def replace_w(x):
            if x.op.name.endswith('kernel'):
                return bit_utils.quantize_w(tf.tanh(x), bit=self._w_bit)
            else:
                return x

        with bit_utils.replace_variable(replace_w):
            if self._gate_linear is None:
                bias_ones = self._bias_initializer
                if self._bias_initializer is None:
                    bias_ones = tf.constant_initializer(
                        1.0, dtype=inputs.dtype)
                with tf.variable_scope("gates"):  # Reset gate and update gate.
                    self._gate_linear = rnn_cell_impl._Linear(
                        [inputs, state],
                        2 * self._num_units,
                        True,
                        bias_initializer=bias_ones,
                        kernel_initializer=self._kernel_initializer)

            value = tf.sigmoid(self._gate_linear([inputs, state]))
            r, u = tf.split(value=value, num_or_size_splits=2, axis=1)

            r_state = bit_utils.round_bit(r * state, bit=self._f_bit)
            if self._candidate_linear is None:
                with tf.variable_scope("candidate"):
                    self._candidate_linear = rnn_cell_impl._Linear(
                        [inputs, r_state],
                        self._num_units,
                        True,
                        bias_initializer=self._bias_initializer,
                        kernel_initializer=self._kernel_initializer)
            c = self._activation(self._candidate_linear([inputs, r_state]))
            c = bit_utils.round_bit(c, bit=self._f_bit)
            new_h = bit_utils.round_bit(
                u * state + (1 - u) * c, bit=self._f_bit)
        return new_h, new_h

class BitLSTMCell(tf.nn.rnn_cell.BasicLSTMCell):
    pass
<|endoftext|>


<commit_before>import bisect
import Image

from cStringIO import StringIO

from blikit.models import BlobObject, TreeObject, ObjectTypeMismatch

def get_image_size(data):
    img = Image.open(StringIO(data))
    return img.size

def calc_thumb_size(data, max_size):
    w, h = get_image_size(data)
    max_w, max_h = max_size

    ratio_w = max_w * 1.0 / w
    ratio_h = max_h * 1.0 / h

    if ratio_w > 1 and ratio_h > 1:
        return (w, h)
    elif ratio_w > ratio_h:
        return (w * ratio_h, max_h)
    else:
        return (max_w, h * ratio_w)

READMES = [
    'README',
    'README.txt',
    'README.rst',
]

def find_readme(tree_obj):
    if not isinstance(tree_obj, TreeObject):
        raise ObjectTypeMismatch

    for readme_name in READMES:
        if readme_name in tree_obj:
            obj = tree_obj[readme_name]
            # TODO: follow symlink
            if isinstance(obj, BlobObject):
                return obj

    return None

def recent_files(odb, count=None, path=None, pattern=None, show_hidden=False):
    '''search recently added files

    return [(datetime.datetime when file added, file_path)]
    '''
    args = ['--date-order']

    if path is None:
        path = '/'
    else:
        git_args.extend(['--', path])

    added_names = set()
    results = []

    for commit_hash in odb.git('log', *args, format='format:%H'):
        commit_obj = odb.get_commit(commit_hash)
        added_in_this_commit = []

        # TODO: if path is not TreeObject
        tree_obj = commit_obj.tree[path]

        if not commit_obj.parents:
            # initial commit
            # all files is added in this commit
            added_in_this_commit = tree_obj.find(type_=BlobObject)

        elif len(commit_obj.parents) == 1:
            parent_commit = commit_obj.parents[0]
            parent_tree = parent_commit.tree[path]
            try:
                a, _, _ = tree_obj.diff(parent_tree)

                for obj in a:
                    if isinstance(obj, TreeObject):
                        added_in_this_commit.extend(obj.find(type_=BlobObject))
                    elif isinstance(obj, BlobObject):
                        added_in_this_commit.append(obj)

            except KeyError:
                # parent commit's tree does not have this path
                # so all files under this path added in this commit
                added_in_this_commit = tree_obj.find(type_=BlobObject)

        elif len(commit_obj.parents) >= 2:
            s = None
            for parent_commit in commit_obj.parents:
                parent_tree = parent_commit.tree[path]
                a, _, _ = tree_obj.diff(parent_tree)

                if s is None:
                    s = set(a)
                else:
                    # files not shown in all parent's tree
                    # is added in this merge commit
                    s.intersection_update(a)

            for obj in s:
                if isinstance(obj, TreeObject):
                    added_in_this_commit.extend(obj.find(type_=BlobObject))
                elif isinstance(obj, BlobObject):
                    added_in_this_commit.append(obj)

        for obj in added_in_this_commit:
            # TODO: use pattern
            # to hide dot directories find "/." in obj.abs_name
            if isinstance(obj, BlobObject) and \
               (show_hidden or '/.' not in obj.abs_name) and \
               obj.abs_name not in added_names:
                added_names.add(obj.abs_name)
                bisect.insort_right(results, (commit_obj.commit_time, obj))

        if count is not None:
            while len(results) > count:
                # remove oldest value
                results.pop(0)

    results.reverse()
    return results
<commit_msg>HEAD にあるファイルだけ残す<commit_after>import bisect
import Image

from cStringIO import StringIO

from blikit.models import BlobObject, TreeObject, ObjectTypeMismatch

def get_image_size(data):
    img = Image.open(StringIO(data))
    return img.size

def calc_thumb_size(data, max_size):
    w, h = get_image_size(data)
    max_w, max_h = max_size

    ratio_w = max_w * 1.0 / w
    ratio_h = max_h * 1.0 / h

    if ratio_w > 1 and ratio_h > 1:
        return (w, h)
    elif ratio_w > ratio_h:
        return (w * ratio_h, max_h)
    else:
        return (max_w, h * ratio_w)

READMES = [
    'README',
    'README.txt',
    'README.rst',
]

def find_readme(tree_obj):
    if not isinstance(tree_obj, TreeObject):
        raise ObjectTypeMismatch

    for readme_name in READMES:
        if readme_name in tree_obj:
            obj = tree_obj[readme_name]
            # TODO: follow symlink
            if isinstance(obj, BlobObject):
                return obj

    return None

def recent_files(odb, count=None, path=None, pattern=None, show_hidden=False):
    '''search recently added files

    return [(datetime.datetime when file added, file_path)]
    '''
    args = ['--date-order']

    if path is None:
        path = '/'
    else:
        git_args.extend(['--', path])

    added_names = set()
    results = []

    for commit_hash in odb.git('log', *args, format='format:%H'):
        commit_obj = odb.get_commit(commit_hash)
        added_in_this_commit = []

        # TODO: if path is not TreeObject
        tree_obj = commit_obj.tree[path]

        if not commit_obj.parents:
            # initial commit
            # all files is added in this commit
            added_in_this_commit = tree_obj.find(type_=BlobObject)

        elif len(commit_obj.parents) == 1:
            parent_commit = commit_obj.parents[0]
            parent_tree = parent_commit.tree[path]
            try:
                a, _, _ = tree_obj.diff(parent_tree)

                for obj in a:
                    if isinstance(obj, TreeObject):
                        added_in_this_commit.extend(obj.find(type_=BlobObject))
                    elif isinstance(obj, BlobObject):
                        added_in_this_commit.append(obj)

            except KeyError:
                # parent commit's tree does not have this path
                # so all files under this path added in this commit
                added_in_this_commit = tree_obj.find(type_=BlobObject)

        elif len(commit_obj.parents) >= 2:
            s = None
            for parent_commit in commit_obj.parents:
                parent_tree = parent_commit.tree[path]
                a, _, _ = tree_obj.diff(parent_tree)

                if s is None:
                    s = set(a)
                else:
                    # files not shown in all parent's tree
                    # is added in this merge commit
                    s.intersection_update(a)

            for obj in s:
                if isinstance(obj, TreeObject):
                    added_in_this_commit.extend(obj.find(type_=BlobObject))
                elif isinstance(obj, BlobObject):
                    added_in_this_commit.append(obj)

        for obj in added_in_this_commit:
            # TODO: use pattern
            # to hide dot directories find "/." in obj.abs_name
            if isinstance(obj, BlobObject) and \
               (show_hidden or '/.' not in obj.abs_name) and \
               obj.abs_name not in added_names and \
               obj.abs_name in odb.head.tree:
                added_names.add(obj.abs_name)
                bisect.insort_right(results, (commit_obj.commit_time, obj))

        if count is not None:
            while len(results) > count:
                # remove oldest value
                results.pop(0)

    results.reverse()
    return results
<|endoftext|>


<commit_before>__author__ = 'Will Hart'

import logging
import os.path
import tornado.httpserver
import tornado.ioloop
import tornado.web

from blitz.constants import CommunicationCodes
from blitz.data.database import DatabaseClient
from blitz.communications.boards import BoardManager
import blitz.communications.signals as sigs
from blitz.communications.tcp import TcpCommunicationException, TcpBase
import blitz.web.api as blitz_api
import blitz.web.http as blitz_http

class Config(object):
    """
    Holds configuration for a client application
    """

    settings = {}

    def __init__(self):
        """
        Sets up default settings
        """
        self.settings = {
            "template_path": os.path.join(os.path.dirname(__file__), "templates"),
            "static_path": os.path.join(os.path.dirname(__file__), "static"),
            "database_path": os.path.join(os.path.dirname(__file__), "data", "app.db"),
            "port": 8989,
            "autoescape": None,
            "debug": True
        }

    def get(self, key):
        """
        Gets an item from settings

        :raises: KeyError if the item doesn't exist
        :returns: A value corresponding to the given key
        """
        if key in self.settings.keys():
            return self.settings[key]
        raise KeyError("Unknown configuration setting - " + key)

    def set(self, key, value):
        """
        Sets the given configuration key to value

        :param key: the key to set
        :param value: the value to set the key to
        :returns: the value that was set
        """
        self.settings[key] = value
        return value

    def __getitem__(self, item):
        """
        A custom implementation of dict getters
        """
        return self.get(item)

    def __setitem__(self, key, value):
        """
        A custom implementation of dict setters
        """
        return self.set(key, value)

class BaseApplicationClient(object):
    """
    A basic application which provides access method agnostic functionality
    for running a Blitz client side application.  Can be inherited to run
    web or desktop type applications
    """

    def __init__(self):
        """
        Create a new client web application, setting defaults
        """

        # create a file logger and set it up for logging to file
        logging.basicConfig(filename='client_log.txt', level=logging.DEBUG,
                            format='%(asctime)-27s %(levelname)-10s %(name)-25s %(threadName)-15s   %(message)s')
        ch = logging.StreamHandler()
        self.logger = logging.getLogger(__name__)
        self.logger.addHandler(ch)

        # load configuration
        self.config = Config()

        # create a database connection
        self.data = DatabaseClient(path=self.config['database_path'])
        self.data.clear_errors()
        self.logger.info("Initialised DatabaseClient")

        # create an empty TCP connection
        self.tcp = None

        # create a board manager
        self.board_manager = BoardManager(self.data)

        # save variables for later
        self.config['board_manager'] = self.board_manager

        # set up a cache
        self.cache = self.data.get_cache()
        self.variable_cache = self.data.get_cache_variables()

        # subscribe to signals
        sigs.cache_line_received.connect(self.cache_line_received)
        sigs.client_requested_download.connect(self.send_download_request)

    def run(self):
        """
        Starts the application.  Should be provided by implementation
        """
        pass

    def cache_line_received(self, message):
        """
        Handles receiving a line of information from the logger,
        and writing and parsing this to the temporary cache
        """
        results = self.board_manager.parse_message(message)
        self.update_interface(results)

    def send_download_request(self, session_id):
        """
        Sends a request for downloading a given session ID to the data logger
        """
        self.logger.debug("Handling client download request")

        if self.tcp is None:
            self.logger.debug("Failed to handle client download request - no TCP connection")
            self.data.log_error(
                "Unable to request download for session #%s as the logger is not connected" % session_id)
            return

        # delete old session data
        self.data.clear_session_data(session_id)
        self.tcp.send(CommunicationCodes.composite(CommunicationCodes.Download, session_id))

    def connect_to_logger(self, args=None):
        """
        Handles a connection request from the client and establishes a TCP connection
        with the data logger.  If a connection already exists, close it and reopen
        """

        if self.tcp is None:
            # we are connecting
            self.logger.debug("Created TCP connection at client request")
            try:
                self.tcp = TcpBase("127.0.0.1", 8999)  # TODO get from config
                self.tcp.create_client()
            except TcpCommunicationException as tce:
                self.data.log_error("Communication error with the board - connection closed")
                self.tcp.stop()
                self.tcp = None

        else:
            self.tcp.stop()
            self.logger.debug("Closed TCP connection at client request")

            # write a connection error to database for the user
            self.data.log_error("Unable to connect to the network")

    def start_logging(self, args=None):
        """
        Sends a 'start logging' signal to the data logger
        """

        if self.tcp is None:
            self.logger.warning("Attempt to start logging on TCP connection failed - there is no TCP connection")
        else:
            self.logger.debug("Web client requested logging start")
            self.tcp.send(CommunicationCodes.Start)

    def stop_logging(self, args=None):
        """
        Sends a 'stop logging' signal to the data logger
        """

        if self.tcp is None:
            self.logger.warning("Attempt to stop logging on TCP connection failed - there is no TCP connection")

        else:
            self.logger.debug("Web client requested logging stop")
            self.tcp.send(CommunicationCodes.Stop)

    def update_interface(self, data, replace_existing=False):
        """
        Updates the user interface with new cache data received.  This method generates the required data structure,
        the actual interface imlementation is provided by the inheriting class.  Note that this means the inheriting
        class should call `results = super(...).update_interface` to gather the data in the correct format.

        :param data: A list of Cache models or Reading models to convert into a dictionary: {'variable_name': [[x][y]] }
        :param replace_existing: If True, appends to existing cache, if False, replaces cache? Defaults to False

        :returns: The dictionary of readings required to update the UI, or None if no data is found
        """

        result = {}

        for item in data:
            if not item['categoryId'] in result.keys():
                result[item['categoryId']] = [[],[]]  # set up an empty list

            result[item['categoryId']][0].append(item['timeLogged'])
            result[item['categoryId']][1].append(item['value'])

        return None if len(result.keys()) == 0 else result

    def __del__(self):
        """
        A destructor, run when the application is closing
        """
        self.logger.warning("Closing Client Application")

class WebApplicationClient(BaseApplicationClient):
    """
    A basic application which exposes the Api and HTTP request handlers
    provided by Tornado
    """

    io_loop = None

    def __init__(self):
        """
        Create a new client web application, setting defaults
        """

        super(WebApplicationClient, self).__init__()

        # todo remove fixture loading
        try:
            self.data.load_fixtures()
            self.logger.info("Loaded fixtures")
        except Exception:
            pass

        # create an application
        self.application = tornado.web.Application([
                                                   (r'/', blitz_http.IndexHandler),
                                                   (r'/categories', blitz_api.CategoriesHandler),
                                                   (r'/cache', blitz_api.CacheHandler),
                                                   (r'/cache/(?P<since>[^\/]+)', blitz_api.CacheHandler),
                                                   (r'/download/(?P<session_id>[^\/]+)', blitz_api.DownloadHandler),
                                                   (r'/error/(?P<error_id>[^\/]+)', blitz_api.ErrorHandler),
                                                   (r'/session/(?P<session_id>[^\/]+)', blitz_api.SessionHandler),
                                                   (r'/sessions', blitz_api.SessionsHandler),
                                                   (r'/config', blitz_api.ConfigHandler),
                                                   (r'/status', blitz_api.StatusHandler),
                                                   (r'/connect', blitz_http.ConnectHandler),
                                                   (r'/start', blitz_http.StartHandler),
                                                   (r'/stop', blitz_http.StopHandler)
                                                   ], **self.config.settings)

        # create an HTTP server
        self.http_server = tornado.httpserver.HTTPServer(self.application)
        self.logger.debug("HTTPServer __init__")

        # save variables for later
        self.application.settings['socket'] = None
        self.application.settings['data'] = self.data
        self.application.settings['board_manager'] = self.board_manager

    def run(self):
        """
        Starts the application
        """

        # start listening on the configured port and IP
        self.http_server.listen(self.config['port'])
        self.logger.info("HTTP server started listening on port " + str(self.config['port']))

        # start the IO loop
        try:
            self.io_loop = tornado.ioloop.IOLoop.instance()
            self.io_loop.start()
            self.logger.debug("Client HTTP IoLoop exited")

        finally:
            tcp = self.application.settings['socket']
            if tcp is not None:
                tcp.stop()
                self.logger.warning("Stopped TCP Socket")

            self.io_loop.add_callback(self.io_loop.stop)
            self.logger.warning("Stopped IO loop with callback")

    def connect_to_logger(self, args):
        """
        Extend the BaseApplicationClient.connect_to_logger method to save the
        TCP socket to application settings
        """

        # handle the connection through the BaseApplicationClient
        super(WebApplicationClient, self).connect_to_logger(args)

        # save off the socket for later use in the application
        self.application.settings['socket'] = self.tcp

<commit_msg>Force category ID to string for data container<commit_after>__author__ = 'Will Hart'

import logging
import os.path
import tornado.httpserver
import tornado.ioloop
import tornado.web

from blitz.constants import CommunicationCodes
from blitz.data.database import DatabaseClient
from blitz.communications.boards import BoardManager
import blitz.communications.signals as sigs
from blitz.communications.tcp import TcpCommunicationException, TcpBase
import blitz.web.api as blitz_api
import blitz.web.http as blitz_http

class Config(object):
    """
    Holds configuration for a client application
    """

    settings = {}

    def __init__(self):
        """
        Sets up default settings
        """
        self.settings = {
            "template_path": os.path.join(os.path.dirname(__file__), "templates"),
            "static_path": os.path.join(os.path.dirname(__file__), "static"),
            "database_path": os.path.join(os.path.dirname(__file__), "data", "app.db"),
            "port": 8989,
            "autoescape": None,
            "debug": True
        }

    def get(self, key):
        """
        Gets an item from settings

        :raises: KeyError if the item doesn't exist
        :returns: A value corresponding to the given key
        """
        if key in self.settings.keys():
            return self.settings[key]
        raise KeyError("Unknown configuration setting - " + key)

    def set(self, key, value):
        """
        Sets the given configuration key to value

        :param key: the key to set
        :param value: the value to set the key to
        :returns: the value that was set
        """
        self.settings[key] = value
        return value

    def __getitem__(self, item):
        """
        A custom implementation of dict getters
        """
        return self.get(item)

    def __setitem__(self, key, value):
        """
        A custom implementation of dict setters
        """
        return self.set(key, value)

class BaseApplicationClient(object):
    """
    A basic application which provides access method agnostic functionality
    for running a Blitz client side application.  Can be inherited to run
    web or desktop type applications
    """

    def __init__(self):
        """
        Create a new client web application, setting defaults
        """

        # create a file logger and set it up for logging to file
        logging.basicConfig(filename='client_log.txt', level=logging.DEBUG,
                            format='%(asctime)-27s %(levelname)-10s %(name)-25s %(threadName)-15s   %(message)s')
        ch = logging.StreamHandler()
        self.logger = logging.getLogger(__name__)
        self.logger.addHandler(ch)

        # load configuration
        self.config = Config()

        # create a database connection
        self.data = DatabaseClient(path=self.config['database_path'])
        self.data.clear_errors()
        self.logger.info("Initialised DatabaseClient")

        # create an empty TCP connection
        self.tcp = None

        # create a board manager
        self.board_manager = BoardManager(self.data)

        # save variables for later
        self.config['board_manager'] = self.board_manager

        # set up a cache
        self.cache = self.data.get_cache()
        self.variable_cache = self.data.get_cache_variables()

        # subscribe to signals
        sigs.cache_line_received.connect(self.cache_line_received)
        sigs.client_requested_download.connect(self.send_download_request)

    def run(self):
        """
        Starts the application.  Should be provided by implementation
        """
        pass

    def cache_line_received(self, message):
        """
        Handles receiving a line of information from the logger,
        and writing and parsing this to the temporary cache
        """
        results = self.board_manager.parse_message(message)
        self.update_interface(results)

    def send_download_request(self, session_id):
        """
        Sends a request for downloading a given session ID to the data logger
        """
        self.logger.debug("Handling client download request")

        if self.tcp is None:
            self.logger.debug("Failed to handle client download request - no TCP connection")
            self.data.log_error(
                "Unable to request download for session #%s as the logger is not connected" % session_id)
            return

        # delete old session data
        self.data.clear_session_data(session_id)
        self.tcp.send(CommunicationCodes.composite(CommunicationCodes.Download, session_id))

    def connect_to_logger(self, args=None):
        """
        Handles a connection request from the client and establishes a TCP connection
        with the data logger.  If a connection already exists, close it and reopen
        """

        if self.tcp is None:
            # we are connecting
            self.logger.debug("Created TCP connection at client request")
            try:
                self.tcp = TcpBase("127.0.0.1", 8999)  # TODO get from config
                self.tcp.create_client()
            except TcpCommunicationException as tce:
                self.data.log_error("Communication error with the board - connection closed")
                self.tcp.stop()
                self.tcp = None

        else:
            self.tcp.stop()
            self.logger.debug("Closed TCP connection at client request")

            # write a connection error to database for the user
            self.data.log_error("Unable to connect to the network")

    def start_logging(self, args=None):
        """
        Sends a 'start logging' signal to the data logger
        """

        if self.tcp is None:
            self.logger.warning("Attempt to start logging on TCP connection failed - there is no TCP connection")
        else:
            self.logger.debug("Web client requested logging start")
            self.tcp.send(CommunicationCodes.Start)

    def stop_logging(self, args=None):
        """
        Sends a 'stop logging' signal to the data logger
        """

        if self.tcp is None:
            self.logger.warning("Attempt to stop logging on TCP connection failed - there is no TCP connection")

        else:
            self.logger.debug("Web client requested logging stop")
            self.tcp.send(CommunicationCodes.Stop)

    def update_interface(self, data, replace_existing=False):
        """
        Updates the user interface with new cache data received.  This method generates the required data structure,
        the actual interface imlementation is provided by the inheriting class.  Note that this means the inheriting
        class should call `results = super(...).update_interface` to gather the data in the correct format.

        :param data: A list of Cache models or Reading models to convert into a dictionary: {'variable_name': [[x][y]] }
        :param replace_existing: If True, appends to existing cache, if False, replaces cache? Defaults to False

        :returns: The dictionary of readings required to update the UI, or None if no data is found
        """

        result = {}

        for item in data:
            cat_id = str(item['categoryId'])
            if not cat_id in result.keys():
                result[cat_id] = [[],[]]  # set up an empty list

            result[cat_id][0].append(item['timeLogged'])
            result[cat_id][1].append(item['value'])

        return None if len(result.keys()) == 0 else result

    def __del__(self):
        """
        A destructor, run when the application is closing
        """
        self.logger.warning("Closing Client Application")

class WebApplicationClient(BaseApplicationClient):
    """
    A basic application which exposes the Api and HTTP request handlers
    provided by Tornado
    """

    io_loop = None

    def __init__(self):
        """
        Create a new client web application, setting defaults
        """

        super(WebApplicationClient, self).__init__()

        # todo remove fixture loading
        try:
            self.data.load_fixtures()
            self.logger.info("Loaded fixtures")
        except Exception:
            pass

        # create an application
        self.application = tornado.web.Application([
                                                   (r'/', blitz_http.IndexHandler),
                                                   (r'/categories', blitz_api.CategoriesHandler),
                                                   (r'/cache', blitz_api.CacheHandler),
                                                   (r'/cache/(?P<since>[^\/]+)', blitz_api.CacheHandler),
                                                   (r'/download/(?P<session_id>[^\/]+)', blitz_api.DownloadHandler),
                                                   (r'/error/(?P<error_id>[^\/]+)', blitz_api.ErrorHandler),
                                                   (r'/session/(?P<session_id>[^\/]+)', blitz_api.SessionHandler),
                                                   (r'/sessions', blitz_api.SessionsHandler),
                                                   (r'/config', blitz_api.ConfigHandler),
                                                   (r'/status', blitz_api.StatusHandler),
                                                   (r'/connect', blitz_http.ConnectHandler),
                                                   (r'/start', blitz_http.StartHandler),
                                                   (r'/stop', blitz_http.StopHandler)
                                                   ], **self.config.settings)

        # create an HTTP server
        self.http_server = tornado.httpserver.HTTPServer(self.application)
        self.logger.debug("HTTPServer __init__")

        # save variables for later
        self.application.settings['socket'] = None
        self.application.settings['data'] = self.data
        self.application.settings['board_manager'] = self.board_manager

    def run(self):
        """
        Starts the application
        """

        # start listening on the configured port and IP
        self.http_server.listen(self.config['port'])
        self.logger.info("HTTP server started listening on port " + str(self.config['port']))

        # start the IO loop
        try:
            self.io_loop = tornado.ioloop.IOLoop.instance()
            self.io_loop.start()
            self.logger.debug("Client HTTP IoLoop exited")

        finally:
            tcp = self.application.settings['socket']
            if tcp is not None:
                tcp.stop()
                self.logger.warning("Stopped TCP Socket")

            self.io_loop.add_callback(self.io_loop.stop)
            self.logger.warning("Stopped IO loop with callback")

    def connect_to_logger(self, args):
        """
        Extend the BaseApplicationClient.connect_to_logger method to save the
        TCP socket to application settings
        """

        # handle the connection through the BaseApplicationClient
        super(WebApplicationClient, self).connect_to_logger(args)

        # save off the socket for later use in the application
        self.application.settings['socket'] = self.tcp

<|endoftext|>


<commit_before>__author__ = 'Will Hart'

import logging
import threading
import Queue
import zmq

from blitz.io.client_states import *
from blitz.io.server_states import *
import blitz.io.signals as sigs

class TcpCommunicationException(Exception):
    pass

class TcpBase(object):
    REQUEST_TIMEOUT = 2000
    REQUEST_RETRIES = 3
    SERVER_ENDPOINT = "tcp://%s:%s"
    MAX_RESPONSE_LENGTH = 1024

    logger = logging.getLogger(__name__)

    def __init__(self, host="localhost", port=None):
        self.__host = host
        self.__port = port
        self.current_state = None
        self.send_queue = Queue.Queue()
        self.waiting = False
        self.__poller = zmq.Poller()
        self.__stop_event = threading.Event()
        self.__thread = None
        self.__state_machine = None

    def create_client(self, autorun=True):
        self.__context = zmq.Context(1)
        self.__socket = self.__context.socket(zmq.REQ)
        self.__socket.connect(self.SERVER_ENDPOINT % (self.__host, self.__port))
        self.__state_machine = TcpStateMachine(self, self.__stop_event, ClientInitState)

        if autorun:
            self.__run_thread(self.run_client)

    def create_server(self):
        self.__context = zmq.Context(1)
        self.__socket = self.__context.socket(zmq.REP)
        self.__socket.bind(self.SERVER_ENDPOINT % ("*", self.__port))
        self.__state_machine = TcpStateMachine(self, self.__stop_event, ServerIdleState)
        self.__run_thread(self.run_server)

    def __run_thread(self, thread_target):
        self.__poller.register(self.__socket, zmq.POLLIN)
        self.__thread = threading.Thread(target=thread_target, args=[self.__stop_event])
        self.__thread.daemon = True
        self.__thread.start()

    def is_alive(self):
        return not self.__stop_event.is_set()

    def is_logging(self):
        return self.__state_machine is not None and self.__state_machine.is_logging()

    def stop(self):
        self.__stop_event.set()
        if self.__thread is not None:
            self.__thread.join()
        if self.__state_machine is not None:
            self.__state_machine.join()
        self.__stop_event.clear()

    def _do_send(self, message):
        self.send_queue.put(message)

    def send(self, message):
        self.__state_machine.queue_send(message)

    def receive_message(self, message):
        self.__state_machine.queue_receive(message)

    def run_server(self, stop_event):
        self.logger.debug("Starting Server")
        while not stop_event.is_set():
            socks = dict(self.__poller.poll(100))

            if socks.get(self.__socket) == zmq.POLLIN:
                # there is a message to receive messages from clients
                # are not multipart so only one recv call is required
                reply = self.__socket.recv()
                self.receive_message(reply)
                sigs.tcp_message_received.send([self, reply])
                self.logger.info("Server handled receipt of message: %s" % reply)

                # now wait until a response is ready to send
                self.waiting = False
                while not self.waiting:
                    try:
                        response = self.send_queue.get(True, 0.1)
                    except Queue.Empty:
                        if stop_event.is_set():
                            break
                        continue

                    # check if we need to break up the response
                    if len(response) > self.MAX_RESPONSE_LENGTH:
                        parts = [response[i:i + self.MAX_RESPONSE_LENGTH] for i in
                                 range(0, len(response), self.MAX_RESPONSE_LENGTH)]

                        for part in parts:
                            self.__socket.send(part, zmq.SNDMORE)

                        self.__socket.send("")
                    else:
                        self.__socket.send(response)

                    self.waiting = True

        self.__socket.close()
        self.__context.term()
        self.current_state = self.current_state.go_to_state(self, ServerClosedState)
        self.logger.info("Server Closed")

    def run_client(self, stop_event):
        self.logger.info("Client starting")
        while not stop_event.is_set():
            reply = ""
            request = ""
            retries = self.REQUEST_RETRIES
            self.waiting = False

            # read from the send_queue until a message is received
            if not self.waiting:
                try:
                    request = self.send_queue.get(True, 0.1)
                except Queue.Empty:
                    time.sleep(0.1)
                    continue

                self.waiting = True
                self.__socket.send(request)

            # wait for an incoming reply
            while self.waiting:
                socks = dict(self.__poller.poll(self.REQUEST_TIMEOUT))

                # check if we are receiving
                if socks.get(self.__socket) == zmq.POLLIN:
                    # we are receiving - read the bytes
                    reply += self.__socket.recv()

                    if not reply:
                        self.logger.info("Client received empty message")
                        break

                    if not self.__socket.getsockopt(zmq.RCVMORE):
                        self.waiting = False

                else:
                    # nothing was received from the server in the timeout period
                    # reconnect with the socket and then try resending again a
                    # couple of times before just giving up :)
                    # todo disconnect and reconnect the socket
                    # self.__socket.setsockopt(zmq.LINGER, 0)
                    # self.__poller.unregister(self.__socket)
                    # self.__socket.close()

                    retries -= 1

                    if retries <= 0:
                        self.__stop_event.set()
                        self.logger.error(
                            "Unable to send message after %s attempts: %s" % (self.REQUEST_RETRIES, request))
                        raise TcpCommunicationException(
                            "Failed to receive message from client after %s attempts" % self.REQUEST_RETRIES)

                    # otherwise recreate the connection and attempt to resend
                    self.logger.info("Client attempting resend of message %s (#%s)" % (request, retries))
                    # TODO self.create_client(autorun=False)
                    # TODO self.__socket.send(request)

            # now handle the reply
            self.receive_message(reply)
            sigs.tcp_message_received.send([self, reply])
            self.logger.info("Client handled received message: %s" % reply)

        # terminate the context before exiting
        self.__socket.close()
        self.__context.term()
        self.logger.info("Client Closed")

class TcpStateAction(object):
    """
    Manages a TCP action which is queued with the state machine.

    The TcpSTateMachine provides convenience methods for creating and queueing instances
    """

    # command types
    SEND = 0
    RECEIVE = 1

    def __init__(self, command_type, command):
        self.command_type = command_type
        self.command = command

    def is_send(self):
        """Returns true if this is a send command"""
        return self.command_type == TcpStateAction.SEND

class TcpStateMachine(object):
    """
    Manages TCP state and ensures the messages are processed in a thread safe manner, one at a time
    """

    def __init__(self, tcp, stop_event, initial_state):
        self.logger = logging.getLogger(__name__)
        self.__current_state = BaseState().go_to_state(tcp, initial_state)
        self.__commands = Queue.Queue()
        self.__tcp = tcp
        self.__thread = threading.Thread(target=self.run, args=[stop_event])

        # start the state machine thread
        self.__thread.daemon = True
        self.__thread.start()

    def queue_send(self, command):
        """Queues a send message"""
        self.__commands.put(TcpStateAction(TcpStateAction.SEND, command))

    def queue_receive(self, command):
        """Queues a receive message"""
        self.__commands.put(TcpStateAction(TcpStateAction.RECEIVE, command))

    def run(self, stop_event):
        self.logger.info("Starting TCP state machine")
        while not stop_event.is_set():
            # poll queue for messages
            try:
                request = self.__commands.get(True, 0.1)
            except Queue.Empty:
                time.sleep(0.1)
                continue

            # process the request
            if request.is_send():
                self.__current_state = self.__current_state.send_message(self.__tcp, request.command)
            else:
                self.__current_state = self.__current_state.receive_message(self.__tcp, request.command)

        self.logger.info("Stopping TCP state machine")

    def join(self):
        self.__thread.join()

    def is_logging(self):
        return self.__thread.is_alive() and (
            type(self.__current_state) == ClientLoggingState or type(self.__current_state) == ServerLoggingState)
<commit_msg>Tweak debug messages<commit_after>__author__ = 'Will Hart'

import logging
import threading
import Queue
import zmq

from blitz.io.client_states import *
from blitz.io.server_states import *
import blitz.io.signals as sigs

class TcpCommunicationException(Exception):
    pass

class TcpBase(object):
    REQUEST_TIMEOUT = 2000
    REQUEST_RETRIES = 3
    SERVER_ENDPOINT = "tcp://%s:%s"
    MAX_RESPONSE_LENGTH = 1024

    logger = logging.getLogger(__name__)

    def __init__(self, host="localhost", port=None):
        self.__host = host
        self.__port = port
        self.current_state = None
        self.send_queue = Queue.Queue()
        self.waiting = False
        self.__poller = zmq.Poller()
        self.__stop_event = threading.Event()
        self.__thread = None
        self.__state_machine = None

    def create_client(self, autorun=True):
        self.__context = zmq.Context(1)
        self.__socket = self.__context.socket(zmq.REQ)
        self.__socket.connect(self.SERVER_ENDPOINT % (self.__host, self.__port))
        self.__state_machine = TcpStateMachine(self, self.__stop_event, ClientInitState)

        if autorun:
            self.__run_thread(self.run_client)

    def create_server(self):
        self.__context = zmq.Context(1)
        self.__socket = self.__context.socket(zmq.REP)
        self.__socket.bind(self.SERVER_ENDPOINT % ("*", self.__port))
        self.__state_machine = TcpStateMachine(self, self.__stop_event, ServerIdleState)
        self.__run_thread(self.run_server)

    def __run_thread(self, thread_target):
        self.__poller.register(self.__socket, zmq.POLLIN)
        self.__thread = threading.Thread(target=thread_target, args=[self.__stop_event])
        self.__thread.daemon = True
        self.__thread.start()

    def is_alive(self):
        return not self.__stop_event.is_set()

    def is_logging(self):
        return self.__state_machine is not None and self.__state_machine.is_logging()

    def stop(self):
        self.__stop_event.set()
        if self.__thread is not None:
            self.__thread.join()
        if self.__state_machine is not None:
            self.__state_machine.join()
        self.__stop_event.clear()

    def _do_send(self, message):
        self.send_queue.put(message)

    def send(self, message):
        self.__state_machine.queue_send(message)

    def receive_message(self, message):
        self.__state_machine.queue_receive(message)

    def run_server(self, stop_event):
        self.logger.debug("Starting Server")
        while not stop_event.is_set():
            socks = dict(self.__poller.poll(100))

            if socks.get(self.__socket) == zmq.POLLIN:
                # there is a message to receive messages from clients
                # are not multipart so only one recv call is required
                reply = self.__socket.recv()
                self.receive_message(reply)
                sigs.tcp_message_received.send([self, reply])
                self.logger.info("Server processed message: %s" % reply)

                # now wait until a response is ready to send
                self.waiting = False
                while not self.waiting:
                    try:
                        response = self.send_queue.get(True, 0.1)
                    except Queue.Empty:
                        if stop_event.is_set():
                            break
                        continue

                    # check if we need to break up the response
                    if len(response) > self.MAX_RESPONSE_LENGTH:
                        parts = [response[i:i + self.MAX_RESPONSE_LENGTH] for i in
                                 range(0, len(response), self.MAX_RESPONSE_LENGTH)]

                        for part in parts:
                            self.__socket.send(part, zmq.SNDMORE)

                        self.__socket.send("")
                    else:
                        self.__socket.send(response)

                    self.waiting = True

        self.__socket.close()
        self.__context.term()
        self.current_state = self.current_state.go_to_state(self, ServerClosedState)
        self.logger.info("Server Closed")

    def run_client(self, stop_event):
        self.logger.info("Client starting")
        while not stop_event.is_set():
            reply = ""
            request = ""
            retries = self.REQUEST_RETRIES
            self.waiting = False

            # read from the send_queue until a message is received
            if not self.waiting:
                try:
                    request = self.send_queue.get(True, 0.1)
                except Queue.Empty:
                    time.sleep(0.1)
                    continue

                self.waiting = True
                self.__socket.send(request)

            # wait for an incoming reply
            while self.waiting:
                socks = dict(self.__poller.poll(self.REQUEST_TIMEOUT))

                # check if we are receiving
                if socks.get(self.__socket) == zmq.POLLIN:
                    # we are receiving - read the bytes
                    reply += self.__socket.recv()

                    if not reply:
                        self.logger.info("Client received empty message")
                        break

                    if not self.__socket.getsockopt(zmq.RCVMORE):
                        self.waiting = False

                else:
                    # nothing was received from the server in the timeout period
                    # reconnect with the socket and then try resending again a
                    # couple of times before just giving up :)
                    # todo disconnect and reconnect the socket
                    # self.__socket.setsockopt(zmq.LINGER, 0)
                    # self.__poller.unregister(self.__socket)
                    # self.__socket.close()

                    retries -= 1

                    if retries <= 0:
                        self.__stop_event.set()
                        self.logger.error(
                            "Unable to send message after %s attempts: %s" % (self.REQUEST_RETRIES, request))
                        raise TcpCommunicationException(
                            "Failed to receive message from client after %s attempts" % self.REQUEST_RETRIES)

                    # otherwise recreate the connection and attempt to resend
                    self.logger.info("Client attempting resend of message %s (#%s)" % (request, retries))
                    # TODO self.create_client(autorun=False)
                    # TODO self.__socket.send(request)

            # now handle the reply
            self.receive_message(reply)
            sigs.tcp_message_received.send([self, reply])
            self.logger.info("Client processed message: %s" % reply)

        # terminate the context before exiting
        self.__socket.close()
        self.__context.term()
        self.logger.info("Client Closed")

class TcpStateAction(object):
    """
    Manages a TCP action which is queued with the state machine.

    The TcpSTateMachine provides convenience methods for creating and queueing instances
    """

    # command types
    SEND = 0
    RECEIVE = 1

    def __init__(self, command_type, command):
        self.command_type = command_type
        self.command = command

    def is_send(self):
        """Returns true if this is a send command"""
        return self.command_type == TcpStateAction.SEND

class TcpStateMachine(object):
    """
    Manages TCP state and ensures the messages are processed in a thread safe manner, one at a time
    """

    def __init__(self, tcp, stop_event, initial_state):
        self.logger = logging.getLogger(__name__)
        self.__current_state = BaseState().go_to_state(tcp, initial_state)
        self.__commands = Queue.Queue()
        self.__tcp = tcp
        self.__thread = threading.Thread(target=self.run, args=[stop_event])

        # start the state machine thread
        self.__thread.daemon = True
        self.__thread.start()

    def queue_send(self, command):
        """Queues a send message"""
        self.__commands.put(TcpStateAction(TcpStateAction.SEND, command))

    def queue_receive(self, command):
        """Queues a receive message"""
        self.__commands.put(TcpStateAction(TcpStateAction.RECEIVE, command))

    def run(self, stop_event):
        self.logger.info("Starting TCP state machine")
        while not stop_event.is_set():
            # poll queue for messages
            try:
                request = self.__commands.get(True, 0.1)
            except Queue.Empty:
                time.sleep(0.1)
                continue

            # process the request
            if request.is_send():
                self.__current_state = self.__current_state.send_message(self.__tcp, request.command)
            else:
                self.__current_state = self.__current_state.receive_message(self.__tcp, request.command)

        self.logger.info("Stopping TCP state machine")

    def join(self):
        self.__thread.join()

    def is_logging(self):
        return self.__thread.is_alive() and (
            type(self.__current_state) == ClientLoggingState or type(self.__current_state) == ServerLoggingState)
<|endoftext|>


<commit_before>import os
import subprocess as sp
import sys
def getExisting(var):
    try:
        return  os.environ[var]
    except:
        return ''

def dedup(lin):
    l = [i for i in lin]
    for x in list(set(l)):
        i = l.index(x)+1
        while True:
            try:
                index = l[i:].index(x)
            except ValueError:
                break
            l.pop(i+index)

    return l

def gen_cmd(prefix):
    #prefix = os.environ['HOME'] + os.path.sep + '.prefix'
    prefix = os.path.abspath(prefix)
    dirs = sorted([i for i in os.listdir(prefix) if not i.startswith('.') and os.path.isdir(os.path.join(prefix, i))])

    det = dict()
    det['PATH'] = ['bin']
    det['LIBRARY_PATH'] = ['lib', 'lib64']
    det['LD_LIBRARY_PATH'] = det['LIBRARY_PATH']
    det['C_INCLUDE_PATH'] = ['include']
    det['CPLUS_INCLUDE_PATH'] = det['C_INCLUDE_PATH']

    cmds = dict()

    for install in dirs:
        root = os.path.join(prefix, install)
        for var, l in det.items():
            for i in l:
                path = os.path.join(prefix, install, i)
                cmds[var] = cmds.get(var, '') + path + ':'
            cmds[var] = cmds.get(var, '') + getExisting(var)
            l = cmds[var].split(':')
            l = dedup(l)
            cmds[var] = ''.join(map(lambda x: x+':', l)).strip(':')

    cmd = ''
    for var, l in cmds.items():
        cmd += var + '=' + l + ' '

    return 'export ' + cmd

def main():
    try:
        cmd = gen_cmd(sys.argv[1])
    except:
        return

    print cmd

if __name__ == '__main__':
    main()
<commit_msg>This script could use cleaning up.<commit_after>import os
import subprocess as sp
import sys
def getExisting(var):
    try:
        return  os.environ[var]
    except:
        return ''

def dedup(lin):
    l = [i for i in lin]
    for x in list(set(l)):
        i = l.index(x)+1
        while True:
            try:
                index = l[i:].index(x)
            except ValueError:
                break
            l.pop(i+index)

    return l

def gen_cmd(prefix):
    prefix = os.path.abspath(prefix)
    dirs = sorted([i for i in os.listdir(prefix) if not i.startswith('.') and os.path.isdir(os.path.join(prefix, i))])

    det = dict()
    det['PATH'] = ['bin']
    det['LIBRARY_PATH'] = ['lib', 'lib64']
    det['LD_LIBRARY_PATH'] = det['LIBRARY_PATH']
    det['C_INCLUDE_PATH'] = ['include']
    det['CPLUS_INCLUDE_PATH'] = det['C_INCLUDE_PATH']

    cmds = dict()

    for install in dirs:
        root = os.path.join(prefix, install)
        for var, l in det.items():
            for i in l:
                path = os.path.join(prefix, install, i)
                cmds[var] = cmds.get(var, '') + path + ':'

    for var, l in cmds.items():
        cmds[var] = cmds.get(var, '') + getExisting(var)
        l = cmds[var].strip(':').split(':')
        l = dedup(l)
        cmd = ''.join(map(lambda x: x+':', l)).strip(':')
        cmds[var] = cmd

    cmd = ''
    for var, l in cmds.items():
        cmd += var + '=' + l + ' '

    return 'export ' + cmd

def main():
    try:
        cmd = gen_cmd(sys.argv[1])
    except:
        return

    print cmd

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>#!/usr/bin/python

import sys

from sqlalchemy import desc, literal_column

from skylines.config import environment
from skylines import model

NULL = literal_column(str(0))

environment.load_from_file()

tokens = sys.argv[1:]

session = model.DBSession

def get_query(model, query_attr, tokens):
    query_attr = getattr(model, query_attr)

    col_vals = []

    # Matches token exactly
    col_vals.extend([(query_attr, '{}'.format(token), len(token) * 5) for token in tokens])
    # Begins with token
    col_vals.extend([(query_attr, '{}%'.format(token), len(token) * 3) for token in tokens])
    # Has token at word start
    col_vals.extend([(query_attr, '% {}%'.format(token), len(token) * 2) for token in tokens])
    # Has token
    col_vals.extend([(query_attr, '%{}%'.format(token), len(token)) for token in tokens])

    weight = sum([col.weighted_ilike(val, rel) for col, val, rel in col_vals], NULL)

    # The search result table
    table = literal_column('\'{}\''.format(model.__tablename__))

    return session.query(table.label('table'),
                         model.id.label('id'),
                         query_attr.label('name'),
                         weight.label('weight')).filter(weight > NULL)

def search_query(tokens):
    # Escape % and _ properly
    tokens = [t.replace('%', '\%').replace('_', '\_') for t in tokens]

    # Use * as wildcard character
    tokens = [t.replace('*', '%') for t in tokens]

    if len(tokens) > 1:
        tokens.append(' '.join(tokens))

    q1 = get_query(model.User, 'name', tokens)
    q2 = get_query(model.Club, 'name', tokens)
    q3 = get_query(model.Airport, 'name', tokens)

    return q1.union(q2, q3).order_by(desc('weight'))

for u in search_query(tokens).limit(20):
    print u
<commit_msg>search: Simplified token matching creation<commit_after>#!/usr/bin/python

import sys

from sqlalchemy import desc, literal_column

from skylines.config import environment
from skylines import model

NULL = literal_column(str(0))

environment.load_from_file()

tokens = sys.argv[1:]

session = model.DBSession

def get_query(model, query_attr, tokens):
    query_attr = getattr(model, query_attr)

    weighted_ilikes = []

    for token in tokens:
        len_token = len(token)

        def add_pattern(pattern, weight):
            weighted_ilikes.append(
                query_attr.weighted_ilike(pattern.format(token), weight))

        # Matches token exactly
        add_pattern('{}', len_token * 5)
        # Begins with token
        add_pattern('{}%', len_token * 3)
        # Has token at word start
        add_pattern('% {}%', len_token * 2)
        # Has token
        add_pattern('%{}%', len_token)

    weight = sum(weighted_ilikes, NULL)

    # The search result table
    table = literal_column('\'{}\''.format(model.__tablename__))

    return session.query(table.label('table'),
                         model.id.label('id'),
                         query_attr.label('name'),
                         weight.label('weight')).filter(weight > NULL)

def search_query(tokens):
    # Escape % and _ properly
    tokens = [t.replace('%', '\%').replace('_', '\_') for t in tokens]

    # Use * as wildcard character
    tokens = [t.replace('*', '%') for t in tokens]

    if len(tokens) > 1:
        tokens.append(' '.join(tokens))

    q1 = get_query(model.User, 'name', tokens)
    q2 = get_query(model.Club, 'name', tokens)
    q3 = get_query(model.Airport, 'name', tokens)

    return q1.union(q2, q3).order_by(desc('weight'))

for u in search_query(tokens).limit(20):
    print u
<|endoftext|>


<commit_before>import math
import time
import os
import re
from utils import add_cmd, commands, cmd_list, PY3, PrintError
import logging

def chunks(l, n):
    """Yield successive n-sized chunks from l."""
    for i in range(0, len(l), n):
        yield l[i:i+n]

@add_cmd("calc", alias=["math"], minArgs=1)
def calc(bot, event, irc, args):
    """Insert help text here"""
    safe_dict = {
        "sqrt": math.sqrt,
        "pow": math.pow,
        "sin": math.sin,
        "cos": math.cos,
        "tan": math.tan,
        "asin": math.asin,
        "acos": math.acos,
        "atan": math.atan,
        "abs": abs,
        "log": math.log,
        "fact": math.factorial,
        "factorial": math.factorial
    }

    try:
        constant = {
           "e": str(math.e),
           "pi": str(math.pi),
        }

        for c in constant:
            m = args.replace("){0}".format(c), ") * {0}".format(constant[c]))
            p = re.compile(r'([:]?\d*\.\d+|\d+){0}'.format(c))
            subst = "\\1 * " + constant[c]
            m = re.sub(p, subst, m)
            m = re.sub('\\b{0}\\b'.format(c), constant[c], m)

        output = format(eval(m, {"__builtins__": None}, safe_dict), ",d")
        irc.reply(event, "The answer is: {0}".format(output))
    except ArithmeticError:
        irc.reply(event, "\x034Number undefined or too large.")
    except ValueError:
        irc.reply(event, "\x034Invalid Input")

@add_cmd("echo", minArgs=1)
def echo(bot, event, irc, args):
    """Help text"""
    irc.reply(event, args)

@add_cmd("ping", minArgs=0)
def ping(bot, event, irc, args):
    """Help text"""
    irc.reply(event, "PONG!")

@add_cmd("join", admin=True, minArgs=1)
def join(bot, event, irc, args):
    """Help text"""
    irc.join(args)

@add_cmd("part", alias=["leave"], admin=True, minArgs=0)
def part(bot, event, irc, args):
    """Help text"""
    if len(args):
        irc.part(args[0])
    else:
        irc.part(event.target)

@add_cmd("ban", admin=True, minArgs=1)
def ban(bot, event, irc, args):
    """Help text"""
    if len(args):
        if len(args) > 1:
            if args[0].find("#") == -1:
                chunked = chunks(args[1:], 4)
                irc.mode(args[0], chunked, "+" + "b" * len(chunked))
            elif args[0].find("#") != -1:
                chunked = chunks(args, 4)
                irc.mode(event.target, chunked, "+" + "b" * len(chunked))
        else:
            irc.ban(event.target, args[0])
    else:
        irc.ban(event.target, event.source.nick)

@add_cmd("unban", admin=True, minArgs=1)
def unban(bot, event, irc, args):
    """Help text"""
    if len(args):
        if len(args) > 1:
            if args[0].find("#") == -1:
                chunked = chunks(args[1:], 4)
                irc.mode(args[0], chunked, "-" + "b" * len(chunked))
            elif args[0].find("#") != -1:
                chunked = chunks(args, 4)
                irc.mode(event.target, chunked, "-" + "b" * len(chunked))
        else:
            irc.unban(event.target, args[0])
    else:
        irc.unban(event.target, event.source.nick)

@add_cmd("op", admin=True, minArgs=0)
def op(bot, event, irc, args):
    """Help text"""
    if len(args):
        if len(args) > 1 and args[0].find("#") == -1:
            irc.mode(args[0], chunks(args[1:], 4), "+oooo")
        elif len(args) > 1 and args[0].find("#") != -1:
            irc.mode(event.target, chunks(args, 4), "+oooo")
        elif len(args) == 1:
            irc.op(event.target, args[0])
    else:
        irc.op(event.target, event.source.nick)

@add_cmd("deop", admin=True, minArgs=0)
def deop(bot, event, irc, args):
    """Help text"""
    if len(args):
        if len(args) > 1 and args[0].find("#") == -1:
            irc.mode(args[0], chunks(args[1:], 4), "-oooo")
        elif len(args) > 1 and args[0].find("#") != -1:
            irc.mode(event.target, chunks(args, 4), "-oooo")
        elif len(args) == 1:
            irc.deop(event.target, args[0])
    else:
        irc.deop(event.target, event.source.nick)

@add_cmd("voice", admin=True, minArgs=0)
def voice(bot, event, irc, args):
    if len(args):
        if len(args) > 1 and args[0].find("#") == -1:
            irc.mode(args[0], chunks(args[1:], 4), "+vvvv")
        elif len(args) > 1 and args[0].find("#") != -1:
            irc.mode(event.target, chunks(args, 4), "+vvvv")
        elif len(args) == 1:
            irc.deop(event.target, args[0])
    else:
        irc.voice(event.target, event.source.nick)

@add_cmd("unvoice", admin=True, minArgs=0)
def unvoice(bot, event, irc, args):
    if len(args):
        if len(args) > 1 and args[0].find("#") == -1:
            irc.mode(args[0], chunks(args[1:], 4), "-vvvv")
        elif len(args) > 1 and args[0].find("#") != -1:
            irc.mode(event.target, chunks(args, 4), "-vvvv")
        elif len(args) == 1:
            irc.deop(event.target, args[0])
            irc.unvoice(event.target, args[0])
    else:
        irc.unvoice(event.target, event.source.nick)

@add_cmd("nick", owner=True, minArgs=1)
def nick(bot, event, irc, args):
    bot.config['nickname'] = args[0]
    irc.nick(args[0])

@add_cmd("log.level", admin=True, minArgs=1)
def logLevel(bot, event, irc, args):
    if args[0] == "debug":
        level = logging.DEBUG
    elif args[0] == "info":
        level = logging.INFO
    elif args[0] == "error":
        level = logging.ERROR
    elif args[0] == "warning":
        level = logging.WARNING
    elif args[0] == "critical":
        level = logging.CRITICAL
    else:
        level = logging.INFO # Default logging level
        irc.reply(event, "Invalid log level {0}".format(args))
    logging.getLogger().setLevel(level)

@add_cmd("quit", admin=True, minArgs=0)
def Quit(bot, event, irc, args):
    """(\x02quit <text>\x0F) -- Exits the bot with the QUIT message <text>."""
    args = "zIRC - https://github.com/itslukej/zirc" if not args else " ".join(args)
    irc.quit(args)
    time.sleep(1)
    os._exit(0)

@add_cmd("help", minArgs=0)
def Help(bot, event, irc, args):
    """Help text"""
    """Help text"""
        try:
            irc.reply(event, "Usage: {0}".format(commands[args[0]]['function'].__doc__))
        except KeyError:
            try:
                irc.reply(event, "Usage: {0}".format(aliases[args[0]]['function'].__doc__))
            except KeyError:
                irc.reply(event, "Invalid command {0}".format(args[0]))
    else:
        irc.reply(event, "Usage: {0}".format(commands["help"]['function'].__doc__))

@add_cmd("list", minArgs=0, alias=["ls"])
def List(bot, event, irc, args):
    """Help text"""
    irc.reply(event, ", ".join(sorted(cmd_list)))

@add_cmd("reload", admin=True)
def Reload(bot, event, irc, args):
    if len(args) >= 1:
    if PY3:
        reload = __import__("importlib").reload

    if args[0] in ['commands', 'utils']:
        try:
            reload(args)
            irc.reply(event, "Reloaded {0}".format(args))
        except ImportError:
            PrintError()
    else:
        irc.reply(event, "Wrong module name")
<commit_msg>Avoid code duplication in the if's<commit_after>import math
import time
import os
import re
from utils import add_cmd, commands, cmd_list, PY3, PrintError
import logging

def chunks(l, n):
    """Yield successive n-sized chunks from l."""
    for i in range(0, len(l), n):
        yield l[i:i+n]

@add_cmd("calc", alias=["math"], minArgs=1)
def calc(bot, event, irc, args):
    """Insert help text here"""
    safe_dict = {
        "sqrt": math.sqrt,
        "pow": math.pow,
        "sin": math.sin,
        "cos": math.cos,
        "tan": math.tan,
        "asin": math.asin,
        "acos": math.acos,
        "atan": math.atan,
        "abs": abs,
        "log": math.log,
        "fact": math.factorial,
        "factorial": math.factorial
    }

    try:
        constant = {
           "e": str(math.e),
           "pi": str(math.pi),
        }

        for c in constant:
            m = args.replace("){0}".format(c), ") * {0}".format(constant[c]))
            p = re.compile(r'([:]?\d*\.\d+|\d+){0}'.format(c))
            subst = "\\1 * " + constant[c]
            m = re.sub(p, subst, m)
            m = re.sub('\\b{0}\\b'.format(c), constant[c], m)

        output = format(eval(m, {"__builtins__": None}, safe_dict), ",d")
        irc.reply(event, "The answer is: {0}".format(output))
    except ArithmeticError:
        irc.reply(event, "\x034Number undefined or too large.")
    except ValueError:
        irc.reply(event, "\x034Invalid Input")

@add_cmd("echo", minArgs=1)
def echo(bot, event, irc, args):
    """Help text"""
    irc.reply(event, args)

@add_cmd("ping", minArgs=0)
def ping(bot, event, irc, args):
    """Help text"""
    irc.reply(event, "PONG!")

@add_cmd("join", admin=True, minArgs=1)
def join(bot, event, irc, args):
    """Help text"""
    irc.join(args)

@add_cmd("part", alias=["leave"], admin=True, minArgs=0)
def part(bot, event, irc, args):
    """Help text"""
    if len(args):
        irc.part(args[0])
    else:
        irc.part(event.target)

@add_cmd("ban", admin=True, minArgs=1)
def ban(bot, event, irc, args):
    """Help text"""
    if len(args):
        if len(args) > 1:
            if args[0].find("#") == -1:
                chunked = chunks(args[1:], 4)
                irc.mode(args[0], chunked, "+" + "b" * len(chunked))
            elif args[0].find("#") != -1:
                chunked = chunks(args, 4)
                irc.mode(event.target, chunked, "+" + "b" * len(chunked))
        else:
            irc.ban(event.target, args[0])
    else:
        irc.ban(event.target, event.source.nick)

@add_cmd("unban", admin=True, minArgs=1)
def unban(bot, event, irc, args):
    """Help text"""
    if len(args):
        if len(args) > 1:
            if args[0].find("#") == -1:
                chunked = chunks(args[1:], 4)
                irc.mode(args[0], chunked, "-" + "b" * len(chunked))
            elif args[0].find("#") != -1:
                chunked = chunks(args, 4)
                irc.mode(event.target, chunked, "-" + "b" * len(chunked))
        else:
            irc.unban(event.target, args[0])
    else:
        irc.unban(event.target, event.source.nick)

@add_cmd("op", admin=True, minArgs=0)
def op(bot, event, irc, args):
    """Help text"""
    if len(args):
        if len(args) > 1:
            if args[0].find("#") == -1:
                chunked = chunks(args[1:], 4)
                irc.mode(args[0], chunked, "+" + "o" * len(chunked))
            elif args[0].find("#") != -1:
                chunks(args, 4)
                irc.mode(event.target, chunked, "+" + "o" * len(chunked))
        else:
            irc.op(event.target, args[0])
    else:
        irc.op(event.target, event.source.nick)

@add_cmd("deop", admin=True, minArgs=0)
def deop(bot, event, irc, args):
    """Help text"""
    if len(args):
        if len(args) > 1:
            if args[0].find("#") == -1:
                chunked = chunks(args[1:], 4)
                irc.mode(args[0], chunked, "-" + "o" * len(chunked))
            elif args[0].find("#") != -1:
                chunked = chunks(args, 4)
                irc.mode(event.target, chunked, "-" + "o" * len(chunked))
        else:
            irc.deop(event.target, args[0])
    else:
        irc.deop(event.target, event.source.nick)

@add_cmd("voice", admin=True, minArgs=0)
def voice(bot, event, irc, args):
    if len(args):
        if len(args) > 1:
            if args[0].find("#") == -1:
                irc.mode(args[0], chunks(args[1:], 4), "+vvvv")
            elif args[0].find("#") != -1:
                irc.mode(event.target, chunks(args, 4), "+vvvv")
        else:
            irc.deop(event.target, args[0])
    else:
        irc.voice(event.target, event.source.nick)

@add_cmd("unvoice", admin=True, minArgs=0)
def unvoice(bot, event, irc, args):
    if len(args):
        if len(args) > 1:
            if args[0].find("#") == -1:
                irc.mode(args[0], chunks(args[1:], 4), "-vvvv")
            elif args[0].find("#") != -1:
                irc.mode(event.target, chunks(args, 4), "-vvvv")
        else:
            irc.deop(event.target, args[0])
            irc.unvoice(event.target, args[0])
    else:
        irc.unvoice(event.target, event.source.nick)

@add_cmd("nick", owner=True, minArgs=1)
def nick(bot, event, irc, args):
    bot.config['nickname'] = args[0]
    irc.nick(args[0])

@add_cmd("log.level", admin=True, minArgs=1)
def logLevel(bot, event, irc, args):
    if args[0] == "debug":
        level = logging.DEBUG
    elif args[0] == "info":
        level = logging.INFO
    elif args[0] == "error":
        level = logging.ERROR
    elif args[0] == "warning":
        level = logging.WARNING
    elif args[0] == "critical":
        level = logging.CRITICAL
    else:
        level = logging.INFO # Default logging level
        irc.reply(event, "Invalid log level {0}".format(args))
    logging.getLogger().setLevel(level)

@add_cmd("quit", admin=True, minArgs=0)
def Quit(bot, event, irc, args):
    """(\x02quit <text>\x0F) -- Exits the bot with the QUIT message <text>."""
    args = "zIRC - https://github.com/itslukej/zirc" if not args else " ".join(args)
    irc.quit(args)
    time.sleep(1)
    os._exit(0)

@add_cmd("help", minArgs=0)
def Help(bot, event, irc, args):
    """Help text"""
    """Help text"""
        try:
            irc.reply(event, "Usage: {0}".format(commands[args[0]]['function'].__doc__))
        except KeyError:
            try:
                irc.reply(event, "Usage: {0}".format(aliases[args[0]]['function'].__doc__))
            except KeyError:
                irc.reply(event, "Invalid command {0}".format(args[0]))
    else:
        irc.reply(event, "Usage: {0}".format(commands["help"]['function'].__doc__))

@add_cmd("list", minArgs=0, alias=["ls"])
def List(bot, event, irc, args):
    """Help text"""
    irc.reply(event, ", ".join(sorted(cmd_list)))

@add_cmd("reload", admin=True)
def Reload(bot, event, irc, args):
    if len(args) >= 1:
    if PY3:
        reload = __import__("importlib").reload

    if args[0] in ['commands', 'utils']:
        try:
            reload(args)
            irc.reply(event, "Reloaded {0}".format(args))
        except ImportError:
            PrintError()
    else:
        irc.reply(event, "Wrong module name")
<|endoftext|>