<commit_before>from smpp.pdu_builder import SubmitSM

from vumi.log import log
from vumi.utils import get_operator_number
from vumi.transports.smpp import SmppTransport
from vumi.transports.smpp.clientserver.client import (
    EsmeTransceiver, EsmeTransceiverFactory)

class EnhancedSmppTransport(SmppTransport):

    DELIVERY_REPORT_STATUS_MAPPING = {
        # Output values should map to themselves:
        'delivered': 'delivered',
        'failed': 'failed',
        'pending': 'pending',
        # SMPP `message_state` values:
        'ENROUTE': 'pending',
        'DELIVERED': 'delivered',
        'EXPIRED': 'failed',
        'DELETED': 'failed',
        'UNDELIVERABLE': 'failed',
        'ACCEPTED': 'delivered',
        'UNKNOWN': 'pending',
        'REJECTED': 'failed',
        # From the most common regex-extracted format:
        'DELIVRD': 'delivered',
        'REJECTD': 'failed'}

    def setup_transport(self):
        self.submit_sm_encoding = self.config.get("submit_sm_encoding", 'utf-8')
        self.submit_sm_data_encoding = self.config.get("submit_sm_data_encoding", 0)
        super(EnhancedSmppTransport, self).setup_transport()

    def make_factory(self):
        return EnhancedEsmeTransceiverFactory(
            self.client_config,
            self.r_server,
            self.esme_callbacks)

    def delivery_status(self, stat):
        return self.DELIVERY_REPORT_STATUS_MAPPING.get('stat', pending)

    def send_smpp(self, message):
        log.msg("Sending SMPP message: %s" % (message))
        # first do a lookup in our YAML to see if we've got a source_addr
        # defined for the given MT number, if not, trust the from_addr
        # in the message
        to_addr = message['to_addr']
        from_addr = message['from_addr']
        text = message['content']
        route = get_operator_number(to_addr,
                self.config.get('COUNTRY_CODE', ''),
                self.config.get('OPERATOR_PREFIX', {}),
                self.config.get('OPERATOR_NUMBER', {})) or from_addr
        sequence_number = self.esme_client.submit_sm(
                short_message=text.encode(self.submit_sm_encoding),
                data_coding=self.submit_sm_data_encoding,
                destination_addr=str(to_addr),
                source_addr=route)
        return sequence_number

class EnhancedEsmeTransceiverFactory(EsmeTransceiverFactory):

    def buildProtocol(self, addr):
        log.msg('Connected')
        self.esme = EnhancedEsmeTransceiver(self.config, self.kvs, self.esme_callbacks)
        self.resetDelay()
        return self.esme

class EnhancedEsmeTransceiver(EsmeTransceiver):

    def submit_sm(self, **kwargs):
        if self.state not in ['BOUND_TX', 'BOUND_TRX']:
            log.err(('WARNING: submit_sm in wrong state: %s, '
                            'dropping message: %s' % (self.state, kwargs)))
            return 0
        else:
            sequence_number = self.get_seq()
            pdu = SubmitSM(sequence_number, **dict(self.defaults, **kwargs))
            self.get_next_seq()

            message = kwargs['short_message']
            if len(message) > 254:
                pdu.add_message_payload(''.join('%02x' % ord(c) for c in message))

            self.send_pdu(pdu)
            self.push_unacked(sequence_number)
            return sequence_number
<commit_msg>fix translation between smpp delivery and vusion status<commit_after>from smpp.pdu_builder import SubmitSM

from vumi.log import log
from vumi.utils import get_operator_number
from vumi.transports.smpp import SmppTransport
from vumi.transports.smpp.clientserver.client import (
    EsmeTransceiver, EsmeTransceiverFactory)

class EnhancedSmppTransport(SmppTransport):

    DELIVERY_REPORT_STATUS_MAPPING = {
        # Output values should map to themselves:
        'delivered': 'delivered',
        'failed': 'failed',
        'pending': 'pending',
        # SMPP `message_state` values:
        'ENROUTE': 'pending',
        'DELIVERED': 'delivered',
        'EXPIRED': 'failed',
        'DELETED': 'failed',
        'UNDELIVERABLE': 'failed',
        'ACCEPTED': 'delivered',
        'UNKNOWN': 'pending',
        'REJECTED': 'failed',
        # From the most common regex-extracted format:
        'DELIVRD': 'delivered',
        'REJECTD': 'failed'}

    def setup_transport(self):
        self.submit_sm_encoding = self.config.get("submit_sm_encoding", 'utf-8')
        self.submit_sm_data_encoding = self.config.get("submit_sm_data_encoding", 0)
        super(EnhancedSmppTransport, self).setup_transport()

    def make_factory(self):
        return EnhancedEsmeTransceiverFactory(
            self.client_config,
            self.r_server,
            self.esme_callbacks)

    def delivery_status(self, stat):
        return self.DELIVERY_REPORT_STATUS_MAPPING.get(stat, 'pending')

    def send_smpp(self, message):
        log.msg("Sending SMPP message: %s" % (message))
        # first do a lookup in our YAML to see if we've got a source_addr
        # defined for the given MT number, if not, trust the from_addr
        # in the message
        to_addr = message['to_addr']
        from_addr = message['from_addr']
        text = message['content']
        route = get_operator_number(to_addr,
                self.config.get('COUNTRY_CODE', ''),
                self.config.get('OPERATOR_PREFIX', {}),
                self.config.get('OPERATOR_NUMBER', {})) or from_addr
        sequence_number = self.esme_client.submit_sm(
                short_message=text.encode(self.submit_sm_encoding),
                data_coding=self.submit_sm_data_encoding,
                destination_addr=str(to_addr),
                source_addr=route)
        return sequence_number

class EnhancedEsmeTransceiverFactory(EsmeTransceiverFactory):

    def buildProtocol(self, addr):
        log.msg('Connected')
        self.esme = EnhancedEsmeTransceiver(self.config, self.kvs, self.esme_callbacks)
        self.resetDelay()
        return self.esme

class EnhancedEsmeTransceiver(EsmeTransceiver):

    def submit_sm(self, **kwargs):
        if self.state not in ['BOUND_TX', 'BOUND_TRX']:
            log.err(('WARNING: submit_sm in wrong state: %s, '
                            'dropping message: %s' % (self.state, kwargs)))
            return 0
        else:
            sequence_number = self.get_seq()
            pdu = SubmitSM(sequence_number, **dict(self.defaults, **kwargs))
            self.get_next_seq()

            message = kwargs['short_message']
            if len(message) > 254:
                pdu.add_message_payload(''.join('%02x' % ord(c) for c in message))

            self.send_pdu(pdu)
            self.push_unacked(sequence_number)
            return sequence_number
<|endoftext|>


<commit_before>import numpy

from chainer import cuda
from chainer import function
from chainer.utils import type_check

class BatchNormalization(function.Function):

    """Batch normalization on outputs of linear or convolution functions.

    Args:
        size (int or tuple of ints): Size (or shape) of channel
            dimensions.
        decay (float): Decay rate of moving average.
        eps (float): Epsilon value for numerical stability.
        dtype (numpy.dtype): Type to use in computing.

    See: `Batch Normalization: Accelerating Deep Network Training by Reducing\
          Internal Covariate Shift <http://arxiv.org/abs/1502.03167>`_

    """
    parameter_names = ('gamma',  'beta')
    gradient_names = ('ggamma', 'gbeta')

    def __init__(self, size, decay=0.9, eps=1e-5, dtype=numpy.float32):
        if isinstance(size, tuple):
            self.size = size
        elif isinstance(size, int):
            self.size = (size, )
        else:
            raise TypeError('size must be tuple or int')

        size = numpy.prod(size, dtype=int)
        self.dtype = numpy.dtype(dtype)

        self.avg_mean = numpy.zeros((1, size, 1), dtype=self.dtype)
        self.avg_var = numpy.zeros_like(self.avg_mean)

        self.gamma = numpy.ones_like(self.avg_mean)
        self.ggamma = numpy.empty_like(self.gamma)
        self.beta = numpy.zeros_like(self.avg_mean)
        self.gbeta = numpy.empty_like(self.beta)

        self.decay = decay
        self.N = [0]  # as a reference
        self.eps = eps

    def __call__(self, x, test=False, finetune=False):
        """Invokes the forward propagation of BatchNormalization.

        BatchNormalization accepts additional arguments, which controlls three
        different running mode.

        Args:
            x (Variable): An input variable.
            test (bool): If ``True``, BatchNormalization runs in testing mode;
                it normalizes the input using precomputed statistics.
            finetune (bool): If ``True``, BatchNormalization runs in finetuning
                mode; it accumulates the input array to compute population
                statistics for normalization, and normalizes the input using
                batch statistics.

        If ``test`` and ``finetune`` are both ``False``, then
        BatchNormalization runs in training mode; it computes moving averages
        of mean and variance for evaluation during training, and normalizes the
        input using batch statistics.

        """
        self.use_batch_mean = not test or finetune
        self.is_finetune = finetune
        return function.Function.__call__(self, x)

    def check_type_forward(self, in_types):
        type_check.expect(in_types.size() == 1)
        x_type, = in_types

        self_ = type_check.Variable(self, 'self')
        type_check.expect(
            x_type.dtype == numpy.float32,
            x_type.ndim >= self_.size.__len__() + 1,
            x_type.shape[1:len(self.size)+1] == self_.size
        )

    def start_finetuning(self):
        self.N[0] = numpy.array(0)

    def forward(self, x_orig):
        xp = cuda.get_array_module(*x_orig)
        ldim, cdim, rdim = self._internal_shape(x_orig[0])
        x = x_orig[0].reshape(ldim, cdim, rdim)

        if self.use_batch_mean:
            mean = x.mean(axis=(0, 2), keepdims=True)
            var = x.var(axis=(0, 2), keepdims=True)
            var += self.eps
        else:
            mean = self.avg_mean
            var = self.avg_var

        self.std = xp.sqrt(var, dtype=var.dtype)
        x_mu = x - mean
        self.x_hat = x_mu / self.std
        y = self.gamma * self.x_hat
        y += self.beta

        # Compute exponential moving average
        if self.use_batch_mean:
            if self.is_finetune:
                self.N[0] += 1
                decay = 1. / self.N[0]
            else:
                decay = self.decay

            m = ldim * rdim
            adjust = m / max(m - 1., 1.)  # unbiased estimation
            self.avg_mean *= decay
            self.avg_mean += (1 - decay) * adjust * mean
            self.avg_var *= decay
            self.avg_var += (1 - decay) * adjust * var

        return y.reshape(x_orig[0].shape),

    def backward(self, x_orig, gy):
        # TODO(beam2d): Support backprop on inference mode
        assert self.use_batch_mean and not self.is_finetune

        ldim, cdim, rdim = self._internal_shape(x_orig[0])
        gy = gy[0].reshape(ldim, cdim, rdim)
        m = ldim * rdim

        gbeta = gy.sum(axis=(0, 2), keepdims=True)
        self.gbeta += gbeta

        ggamma = (gy * self.x_hat).sum(axis=(0, 2), keepdims=True)
        self.ggamma += ggamma

        coeff = self.gamma / self.std
        gbeta /= m
        ggamma /= m

        gx = coeff * (gy - self.x_hat * ggamma - gbeta)
        return gx.reshape(x_orig[0].shape),

    def _internal_shape(self, x):
        ldim = x.shape[0]
        cdim = self.gamma.size
        rdim = x.size // (ldim * cdim)
        assert ldim * cdim * rdim == x.size
        return map(numpy.int32, (ldim, cdim, rdim))
<commit_msg>Fix scaling of mean moving average<commit_after>import numpy

from chainer import cuda
from chainer import function
from chainer.utils import type_check

class BatchNormalization(function.Function):

    """Batch normalization on outputs of linear or convolution functions.

    Args:
        size (int or tuple of ints): Size (or shape) of channel
            dimensions.
        decay (float): Decay rate of moving average.
        eps (float): Epsilon value for numerical stability.
        dtype (numpy.dtype): Type to use in computing.

    See: `Batch Normalization: Accelerating Deep Network Training by Reducing\
          Internal Covariate Shift <http://arxiv.org/abs/1502.03167>`_

    """
    parameter_names = ('gamma',  'beta')
    gradient_names = ('ggamma', 'gbeta')

    def __init__(self, size, decay=0.9, eps=1e-5, dtype=numpy.float32):
        if isinstance(size, tuple):
            self.size = size
        elif isinstance(size, int):
            self.size = (size, )
        else:
            raise TypeError('size must be tuple or int')

        size = numpy.prod(size, dtype=int)
        self.dtype = numpy.dtype(dtype)

        self.avg_mean = numpy.zeros((1, size, 1), dtype=self.dtype)
        self.avg_var = numpy.zeros_like(self.avg_mean)

        self.gamma = numpy.ones_like(self.avg_mean)
        self.ggamma = numpy.empty_like(self.gamma)
        self.beta = numpy.zeros_like(self.avg_mean)
        self.gbeta = numpy.empty_like(self.beta)

        self.decay = decay
        self.N = [0]  # as a reference
        self.eps = eps

    def __call__(self, x, test=False, finetune=False):
        """Invokes the forward propagation of BatchNormalization.

        BatchNormalization accepts additional arguments, which controlls three
        different running mode.

        Args:
            x (Variable): An input variable.
            test (bool): If ``True``, BatchNormalization runs in testing mode;
                it normalizes the input using precomputed statistics.
            finetune (bool): If ``True``, BatchNormalization runs in finetuning
                mode; it accumulates the input array to compute population
                statistics for normalization, and normalizes the input using
                batch statistics.

        If ``test`` and ``finetune`` are both ``False``, then
        BatchNormalization runs in training mode; it computes moving averages
        of mean and variance for evaluation during training, and normalizes the
        input using batch statistics.

        """
        self.use_batch_mean = not test or finetune
        self.is_finetune = finetune
        return function.Function.__call__(self, x)

    def check_type_forward(self, in_types):
        type_check.expect(in_types.size() == 1)
        x_type, = in_types

        self_ = type_check.Variable(self, 'self')
        type_check.expect(
            x_type.dtype == numpy.float32,
            x_type.ndim >= self_.size.__len__() + 1,
            x_type.shape[1:len(self.size)+1] == self_.size
        )

    def start_finetuning(self):
        self.N[0] = numpy.array(0)

    def forward(self, x_orig):
        xp = cuda.get_array_module(*x_orig)
        ldim, cdim, rdim = self._internal_shape(x_orig[0])
        x = x_orig[0].reshape(ldim, cdim, rdim)

        if self.use_batch_mean:
            mean = x.mean(axis=(0, 2), keepdims=True)
            var = x.var(axis=(0, 2), keepdims=True)
            var += self.eps
        else:
            mean = self.avg_mean
            var = self.avg_var

        self.std = xp.sqrt(var, dtype=var.dtype)
        x_mu = x - mean
        self.x_hat = x_mu / self.std
        y = self.gamma * self.x_hat
        y += self.beta

        # Compute exponential moving average
        if self.use_batch_mean:
            if self.is_finetune:
                self.N[0] += 1
                decay = 1. / self.N[0]
            else:
                decay = self.decay

            m = ldim * rdim
            adjust = m / max(m - 1., 1.)  # unbiased estimation
            self.avg_mean *= decay
            self.avg_mean += (1 - decay) * mean
            self.avg_var *= decay
            self.avg_var += (1 - decay) * adjust * var

        return y.reshape(x_orig[0].shape),

    def backward(self, x_orig, gy):
        # TODO(beam2d): Support backprop on inference mode
        assert self.use_batch_mean and not self.is_finetune

        ldim, cdim, rdim = self._internal_shape(x_orig[0])
        gy = gy[0].reshape(ldim, cdim, rdim)
        m = ldim * rdim

        gbeta = gy.sum(axis=(0, 2), keepdims=True)
        self.gbeta += gbeta

        ggamma = (gy * self.x_hat).sum(axis=(0, 2), keepdims=True)
        self.ggamma += ggamma

        coeff = self.gamma / self.std
        gbeta /= m
        ggamma /= m

        gx = coeff * (gy - self.x_hat * ggamma - gbeta)
        return gx.reshape(x_orig[0].shape),

    def _internal_shape(self, x):
        ldim = x.shape[0]
        cdim = self.gamma.size
        rdim = x.size // (ldim * cdim)
        assert ldim * cdim * rdim == x.size
        return map(numpy.int32, (ldim, cdim, rdim))
<|endoftext|>


<commit_before>import numpy
import six

from chainer import cuda
from chainer import function
from chainer.utils import type_check

class EmbedIDFunction(function.Function):

    def check_type_forward(self, in_types):
        type_check.expect(in_types.size() == 2)
        x_type, w_type = in_types
        type_check.expect(
            x_type.dtype == numpy.int32,
            x_type.ndim >= 1,
        )
        type_check.expect(
            w_type.dtype == numpy.float32,
            w_type.ndim == 2
        )

    def forward(self, inputs):
        x, W = inputs
        return W.take(x, axis=0),

    def backward(self, inputs, grad_outputs):
        xp = cuda.get_array_module(*inputs)
        x, W = inputs
        gy = grad_outputs[0]
        gW = xp.zeros_like(W)

        if xp is numpy:
            # It is equivalent to `numpy.add.at(gW, x, gy)` but ufunc.at is
            # too slow.
            for ix, igy in six.moves.zip(x.reshape((x.size,)),
                                         gy.reshape((x.size, gy.shape[-1]))):
                gW[ix] += igy
        else:
            cuda.elementwise(
                'T gy, int32 x, int32 n_out', 'raw T gW',
                'int w_ind[] = {x, i % n_out}; atomicAdd(&gW[w_ind], gy)',
                'embed_id_bwd')(
                    gy, xp.expand_dims(x, -1), gW.shape[1], gW)
        return None, gW

def embed_id(x, W):
    """Efficient linear function for one-hot input.

    This function implements so called *word embedding*. It takes two
    arguments: a set of IDs (words) ``x`` in :math:`B` dimensional integer
    vector, and a set of all ID (word) embeddings ``W`` in :math:`V \\times d`
    float32 matrix. It outputs :math:`B \\times d` matrix whose ``i``-th
    column is the ``x[i]``-th column of ``W``.

    This function is only differentiable on the input ``W``.

    Args:
        x (~chainer.Variable): Input variable with one-hot representation.
        W (~chainer.Variable): Representation of each ID (a.k.a.
            word embeddings).

    Returns:
        ~chainer.Variable: Output variable.

    .. seealso:: :class:`EmbedID`

    """
    return EmbedIDFunction()(x, W)
<commit_msg>Simplify reshape<commit_after>import numpy
import six

from chainer import cuda
from chainer import function
from chainer.utils import type_check

class EmbedIDFunction(function.Function):

    def check_type_forward(self, in_types):
        type_check.expect(in_types.size() == 2)
        x_type, w_type = in_types
        type_check.expect(
            x_type.dtype == numpy.int32,
            x_type.ndim >= 1,
        )
        type_check.expect(
            w_type.dtype == numpy.float32,
            w_type.ndim == 2
        )

    def forward(self, inputs):
        x, W = inputs
        return W.take(x, axis=0),

    def backward(self, inputs, grad_outputs):
        xp = cuda.get_array_module(*inputs)
        x, W = inputs
        gy = grad_outputs[0]
        gW = xp.zeros_like(W)

        if xp is numpy:
            # It is equivalent to `numpy.add.at(gW, x, gy)` but ufunc.at is
            # too slow.
            for ix, igy in six.moves.zip(x.reshape(-1),
                                         gy.reshape((x.size, -1))):
                gW[ix] += igy
        else:
            cuda.elementwise(
                'T gy, int32 x, int32 n_out', 'raw T gW',
                'int w_ind[] = {x, i % n_out}; atomicAdd(&gW[w_ind], gy)',
                'embed_id_bwd')(
                    gy, xp.expand_dims(x, -1), gW.shape[1], gW)
        return None, gW

def embed_id(x, W):
    """Efficient linear function for one-hot input.

    This function implements so called *word embedding*. It takes two
    arguments: a set of IDs (words) ``x`` in :math:`B` dimensional integer
    vector, and a set of all ID (word) embeddings ``W`` in :math:`V \\times d`
    float32 matrix. It outputs :math:`B \\times d` matrix whose ``i``-th
    column is the ``x[i]``-th column of ``W``.

    This function is only differentiable on the input ``W``.

    Args:
        x (~chainer.Variable): Input variable with one-hot representation.
        W (~chainer.Variable): Representation of each ID (a.k.a.
            word embeddings).

    Returns:
        ~chainer.Variable: Output variable.

    .. seealso:: :class:`EmbedID`

    """
    return EmbedIDFunction()(x, W)
<|endoftext|>


<commit_before>#!/usr/bin/env python
import decimal
import math
import os
from scipy import stats

from django.db import connections, transaction
from numpy import arange
import numpy
import pylab

def hourly_sm_reading_histogram():
    """Creates a histogram of smart meter readings to identify their
    distribution and find outliers.
    """
    cursor = connections["ldc"].cursor()
    cursor.execute("select smr.Reading "
                   "from essex_annotated.SmartMeterReadings smr "
                   "inner join Meters m on m.MeterID = smr.MeterID "
                   "  and m.`Phase` = 1 "
                   "left join meter_data_quality_flag mdqf "
                   "  on smr.MeterID = mdqf.MeterID "
                   "  and mdqf.applicability_start_date <= '2011-05-01' "
                   "  and mdqf.applicability_end_date >= '2012-10-31' "
                   "where smr.MeterID >= 20000 and smr.MeterID < 21000 "
                   "and mdqf.MeterID is null "
                   "and smr.ReadDate >= '2011-05-01' "
                   "and smr.ReadDate <= '2011-10-31'")
    # Create a list of hourly reading values
    readings = [row[0] for row in cursor.fetchall()]
    readings.sort()
    trim = 0.0005
    sample_size = len(readings)
    print("readings length pre-trim", len(readings))
    readings = stats.trimboth(readings, proportiontocut=trim)
    print("readings length post-trim", len(readings))
    bin_sz = 0.1
    bin_min = 0.0
    bin_max = round(max(readings), 1)
    print("bin size=" + str(bin_sz) +
          ", bin min=" + str(bin_min) +
          ", bin max=" + str(bin_max))
    # Create histogram of values
    n, bins, patches = pylab.hist(readings,
                                  bins=pylab.frange(bin_min, bin_max, bin_sz),
                                  normed=False,
                                  histtype="stepfilled")
    pylab.setp(patches, "facecolor", "g", "alpha", 0.75)
    pylab.title("Histogram of Hourly Readings \n"
                "(Summer 2011, sample: " + str(sample_size) + " readings, "
                "trim:" + str(trim * 100.0) + "%)")
    pylab.xlabel("Hourly Reading (kW)")
    pylab.ylabel("Occurrences")

    if trim > 0:
        pylab.savefig("hourly_sm_reading_histogram_summer2011_trimmed.png")
    else:
        pylab.savefig("hourly_sm_reading_histogram_summer2011.png")
    #pylab.show()

def reading_count_histogram():
    """Gets the number of readings in a time range grouped by MeterID"""
    cursor = connections["ldc"].cursor()
    cursor.execute("select count(smr.read_datetime) "
                   "from essex_annotated.SmartMeterReadings smr "
                   "inner join Meters m "
                   "  on m.MeterID = smr.MeterID and m.`Phase` = 1 "
                   "left join meter_data_quality_flag mdqf "
                   "  on smr.MeterID = mdqf.MeterID "
                   "  and mdqf.applicability_start_date <= '2011-05-01' "
                   "  and mdqf.applicability_end_date >= '2012-10-31' "
                   "where smr.MeterID >= 20000 and smr.MeterID < 30000 "
                   "and mdqf.MeterID is null "
                   "and smr.ReadDate >= '2011-05-01' "
                   "and smr.ReadDate <= '2011-10-31' "
                   "group by smr.MeterID")
    # Create a list of observation counts
    reading_counts = [row[0] for row in cursor.fetchall()]
    reading_counts.sort()
    trim = 0.0005
    sample_size = len(reading_counts)
    print("readings length pre-trim", len(reading_counts))
    readings = stats.trimboth(reading_counts, proportiontocut=trim)
    print("readings length post-trim", len(readings))
    bin_sz = 1
    bin_min = min(readings)
    bin_max = max(readings)
    print("bin size=" + str(bin_sz) +
          ", bin min=" + str(bin_min) +
          ", bin max=" + str(bin_max))
    # Create histogram of values
    n, bins, patches = pylab.hist(readings,
                                  bins=range(bin_min, bin_max, bin_sz),
                                  normed=False,
                                  histtype="stepfilled")
    pylab.setp(patches, "facecolor", "g", "alpha", 0.75)
    pylab.title("Number of Readings Histogram \n"
                "(Summer 2011, sample: " + str(sample_size) + " meters, "
                "trim:" + str(trim * 100.0) + "%)")
    pylab.xlabel("Number of Readings During Timeframe")
    pylab.ylabel("Number of Meters")
    if trim > 0:
        pylab.savefig("sm_reading_count_histogram_summer2011_trimmed.png")
    else:
        pylab.savefig("sm_reading_count_histogram_summer2011.png")
    pylab.tick_params(axis='both', )
    pylab.show()

def main():
    """Main method for running univariate outlier analysis functions."""
    os.environ.setdefault("DJANGO_SETTINGS_MODULE", "ldc_analysis.settings")
    # hourly_sm_reading_histogram()
    reading_count_histogram()

if __name__ == '__main__':
    main()<commit_msg>Many changes to univariate analysis for a new lab report.<commit_after>#!/usr/bin/env python
import decimal
import math
import os
from scipy import stats

from django.db import connections, transaction
from numpy import arange
import numpy
import pylab

def create_histogram(a, trim_p, bin_size, p_title, p_ylabel, p_xlabel,
                     file_prefix, dec_prec=0, trim_type="both"):
    """Interacts with pylab to draw and save histogram plot.

    Arguments:
    a -- array_like list of values to plot
    trim_p -- Percentile (range 0 to 1) of values to trim (float)
    bin_size -- Size of histogram's value bins (float)
    p_title -- Title of plot
    p_ylabel -- ylabel of plot
    p_xlabel -- xlabel of plot
    file_prefix -- Filename prefix
    dec_prec -- (Optional) Decimal precision of bins. Defaults to 0. (int)
    trim_type -- (Optional) Controls the tail of distribution that percentile
                 trim_p is applied. Values include "both", "left", and "right".
                 Defaults to "both".
    """
    a.sort()
    sample_size = len(a)
    print("a length pre-trim_p", len(a))
    if trim_type == "left" or trim_type == "right":
        a = stats.trim1(a, trim_p, trim_type)
    else:
        a = stats.trimboth(a, trim_p)
    print("a length post-trim_p", len(a))
    bin_min = math.floor(min(a)) # TODO Round down to dec_prec instead
    bin_max = round(max(a), dec_prec)
    print("bin size=" + str(bin_size) +
          ", bin min=" + str(bin_min) +
          ", bin max=" + str(bin_max))
    # Create histogram of values
    n, bins, patches = pylab.hist(a,
                                  bins=pylab.frange(bin_min,
                                                    bin_max,
                                                    bin_size),
                                  normed=False,
                                  histtype="stepfilled")
    pylab.setp(patches, "facecolor", "g", "alpha", 0.75)
    pylab.title(p_title)
    pylab.xlabel(p_xlabel)
    pylab.ylabel(p_ylabel)

    if trim_p > 0:
        pylab.savefig(file_prefix + "_trimmed.png")
    else:
        pylab.savefig(file_prefix + ".png")
    pylab.show()

def hourly_sm_reading_histogram():
    """Creates a histogram of smart meter readings to identify their
    distribution and find outliers.
    """
    cursor = connections["ldc"].cursor()
    cursor.execute("select smr.Reading "
                   "from essex_annotated.SmartMeterReadings smr "
                   "inner join Meters m on m.MeterID = smr.MeterID "
                   "   and m.`Phase` = 1 "
                   "left join meter_data_quality_flag mdqf "
                   "   on smr.MeterID = mdqf.MeterID "
                   "   and mdqf.applicability_start_date <= '2011-05-01' "
                   "   and mdqf.applicability_end_date >= '2012-10-31' "
                   "where smr.MeterID >= 20000 and smr.MeterID < 20100 "
                   "and mdqf.MeterID is null "
                   "and smr.ReadDate >= '2011-05-01' "
                   "and smr.ReadDate <= '2011-10-31'")
    # Create a list of hourly reading values
    readings = [row[0] for row in cursor.fetchall()]

    # Create histogram
    sample_size = len(readings)
    trim_p = 0.0005
    p_title = "Histogram of Hourly Readings \n" \
                "(Summer 2011, sample: " + str(sample_size) + " readings, " \
                "right trim:" + str(trim_p * 100.0) + "%)"
    p_xlabel = "Hourly Reading (kW)"
    p_ylabel = "Occurrences"
    create_histogram(readings, trim_p, 0.1, p_title, p_ylabel, p_xlabel,
                     "hourly_sm_reading_histogram_summer2011", 1, "right")

def reading_count_histogram():
    """Gets the number of readings in a time range grouped by MeterID"""
    cursor = connections["ldc"].cursor()
    cursor.execute("select count(smr.read_datetime) "
                   "from essex_annotated.SmartMeterReadings smr "
                   "inner join Meters m "
                   "  on m.MeterID = smr.MeterID and m.`Phase` = 1 "
                   "left join meter_data_quality_flag mdqf "
                   "   on smr.MeterID = mdqf.MeterID "
                   "   and mdqf.applicability_start_date <= '2011-05-01' "
                   "   and mdqf.applicability_end_date >= '2012-10-31' "
                   "where smr.MeterID >= 20000 and smr.MeterID < 30000 "
                   "and mdqf.MeterID is null "
                   "and smr.ReadDate >= '2011-05-01' "
                   "and smr.ReadDate <= '2011-10-31' "
                   "group by smr.MeterID")
    # Create a list of observation counts
    reading_counts = [row[0] for row in cursor.fetchall()]

    # Create histogram
    sample_size = len(reading_counts)
    trim_p = 0.001
    p_title = "Number of Readings Histogram \n" \
                "(Summer 2011, sample: " + str(sample_size) + " meters, " \
                "left trim:" + str(trim_p * 100.0) + "%)"
    p_xlabel = "Number of Readings During Timeframe"
    p_ylabel = "Number of Meters"
    create_histogram(reading_counts, trim_p, 1, p_title, p_ylabel, p_xlabel,
                     "sm_reading_count_histogram_summer2011", 0, "left")

def sm_reading_exception_count_histogram():
    """Counts the number of exceptions per MeterNumber and plots a
    histogram.
    """
    cursor = connections["ldc"].cursor()
    cursor.execute("select count(smre.reading_datetime_standard) "
                   "from Meters m "
                   "left join SmartMeterReadingsExceptions smre "
                   "   on smre.MeterNumber = m.MeterNumber "
                   "group by m.MeterNumber")
    # Create a list of observation counts
    exception_counts = [row[0] for row in cursor.fetchall()]

    # Create histogram
    sample_size = len(exception_counts)
    bin_size = 10.0
    trim_p = 0
    p_title = "Number of Exceptions Histogram \n" \
                "(sample: " + str(sample_size) + " meters, " \
                "bin size:" + str(bin_size) + ", " \
                "trim: " + str(trim_p * 100.0) + "%)"
    p_xlabel = "Number of Exception per Meter"
    p_ylabel = "Occurrences"
    create_histogram(exception_counts, trim_p, bin_size, p_title, p_ylabel, p_xlabel,
                     "sm_exception_count_histogram", 0, "right")

def main():
    """Main method for running univariate outlier analysis functions."""
    os.environ.setdefault("DJANGO_SETTINGS_MODULE", "ldc_analysis.settings")
    # hourly_sm_reading_histogram()
    # reading_count_histogram()
    sm_reading_exception_count_histogram()

if __name__ == '__main__':
    main()<|endoftext|>


<commit_before>#!/usr/bin/env python
#-*- coding: utf-8 -*-

import sys
reload(sys)
sys.setdefaultencoding('UTF-8')

import argparse
import re
import subprocess
import time
import urllib
import urllib2

def get_output(language='en', text='Service provided by Baidu', output='out.mp3'):
    """
    1. download mp3 from google tts api
    2. convert it to wav
    3. speed up the wav file, if necessary
    4. convert to mp3
    5. store in some location
    6. return the path
    """
    # generate out.mp3
    google_speak(language, text, output)
    subprocess.Popen(
        "lame --decode out.mp3 out.wav;sox out.wav test.wav speed 1.1;lame test.wav test.mp3;rm test.wav;rm out.wav;rm out.mp3;mpg123 test.mp3", stderr=subprocess.PIPE, shell=True)

def google_speak(language='en', text='Service provided by Baidu', output='out.mp3'):
    '''
    languange = ja, en, pt, zh_CN, ar, th
    '''
    text = text.replace('\n', '')
    text_list = re.split('(\,|\.|\，|\。|、)', text)
    combined_text = []
    for idx, val in enumerate(text_list):
        if idx % 2 == 0:
            combined_text.append(val)
        else:
            joined_text = ''.join((combined_text.pop(), val))
            if len(joined_text) < 100:
                combined_text.append(joined_text)
            else:
                subparts = re.split('( )', joined_text)
                temp_string = ""
                temp_array = []
                for part in subparts:
                    temp_string = temp_string + part
                    if len(temp_string) > 80:
                        temp_array.append(temp_string)
                        temp_string = ""
                # append final part
                temp_array.append(temp_string)
                combined_text.extend(temp_array)

    # Todos
    # rewrite: multi-threaded

    # download chunks and write them to the output file
    if isinstance(output, str):
        output = open(output, 'w')
    for idx, val in enumerate(combined_text):
        print 'transmitting ... %s' % val
        mp3url = "http://translate.google.com/translate_tts?tl=%s&q=%s&total=%s&idx=%s" % (
            language, urllib.quote(val), len(combined_text), idx)
        headers = {"Host": "translate.google.com",
                   "Referer": "http://www.gstatic.com/translate/sound_player2.swf",
                   "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_3) AppleWebKit/535.19 (KHTML, like Gecko) Chrome/18.0.1025.163 Safari/535.19"}
        req = urllib2.Request(mp3url, '', headers)
        sys.stdout.write('.')
        sys.stdout.flush()
        if len(val) > 0:
            try:
                response = urllib2.urlopen(req)
                output.write(response.read())
                time.sleep(.5)
            except urllib2.HTTPError as e:
                print ('%s' % e)

def main():
    description = 'Google TTS Downloader.'
    parser = argparse.ArgumentParser(description=description,
                                     epilog='tunnel snakes rule')

    parser.add_argument('-o', '--output', action='store', nargs='?',
                        help='Filename to output audio to',
                        type=argparse.FileType('w'), default='out.mp3')
    parser.add_argument('-l', '--language', action='store',
                        nargs='?', help='Language to output text to.', default='en')
    group = parser.add_mutually_exclusive_group(required=True)
    group.add_argument(
        '-f', '--file', type=argparse.FileType('r'), help='File to read text from.')
    group.add_argument('-s', '--string', action='store',
                       nargs='+', help='A string of text to convert to speech.')

    if len(sys.argv) == 1:
        parser.print_help()
        sys.exit(1)

    args = parser.parse_args()
    if args.file:
        text = args.file.read()
    if args.string:
        text = ' '.join(map(str, args.string))

    # process text into chunks
    google_speak(args.language, text, args.output)
    args.output.close()

    print('Saved MP3 to %s' % args.output.name)

if __name__ == "__main__":
    main()
<commit_msg>added some todo<commit_after>#!/usr/bin/env python
#-*- coding: utf-8 -*-

import sys
reload(sys)
sys.setdefaultencoding('UTF-8')

import argparse
import re
import subprocess
import time
import urllib
import urllib2

# Todos
# rename the file and variables
# remove accepting command line calls

def output(language='en', text='Service provided by Baidu', output='out.mp3'):
    """
    1. download mp3 from google tts api
    2. convert it to wav
    3. speed up the wav file, if necessary
    4. convert to mp3
    5. store in some location
    6. return the path
    """
    # generate out.mp3
    google_speak(language, text, output)
    subprocess.Popen(
        "lame --decode out.mp3 out.wav;sox out.wav test.wav speed 1.1;lame test.wav test.mp3;rm test.wav;rm out.wav;rm out.mp3;mpg123 test.mp3", stderr=subprocess.PIPE, shell=True)

def google_speak(language='en', text='Service provided by Baidu', output='out.mp3'):
    '''
    languange = ja, en, pt, zh_CN, ar, th
    '''
    text = text.replace('\n', '')
    text_list = re.split('(\,|\.|\，|\。|、)', text)
    combined_text = []
    for idx, val in enumerate(text_list):
        if idx % 2 == 0:
            combined_text.append(val)
        else:
            joined_text = ''.join((combined_text.pop(), val))
            if len(joined_text) < 100:
                combined_text.append(joined_text)
            else:
                subparts = re.split('( )', joined_text)
                temp_string = ""
                temp_array = []
                for part in subparts:
                    temp_string = temp_string + part
                    if len(temp_string) > 80:
                        temp_array.append(temp_string)
                        temp_string = ""
                # append final part
                temp_array.append(temp_string)
                combined_text.extend(temp_array)

    # Todos
    # rewrite: multi-threaded

    # download chunks and write them to the output file
    if isinstance(output, str):
        output = open(output, 'w')
    for idx, val in enumerate(combined_text):
        print 'transmitting ... %s' % val
        mp3url = "http://translate.google.com/translate_tts?tl=%s&q=%s&total=%s&idx=%s" % (
            language, urllib.quote(val), len(combined_text), idx)
        headers = {"Host": "translate.google.com",
                   "Referer": "http://www.gstatic.com/translate/sound_player2.swf",
                   "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_3) AppleWebKit/535.19 (KHTML, like Gecko) Chrome/18.0.1025.163 Safari/535.19"}
        req = urllib2.Request(mp3url, '', headers)
        sys.stdout.write('.')
        sys.stdout.flush()
        if len(val) > 0:
            try:
                response = urllib2.urlopen(req)
                output.write(response.read())
                time.sleep(.5)
            except urllib2.HTTPError as e:
                print ('%s' % e)

def main():
    description = 'Google TTS Downloader.'
    parser = argparse.ArgumentParser(description=description,
                                     epilog='tunnel snakes rule')

    parser.add_argument('-o', '--output', action='store', nargs='?',
                        help='Filename to output audio to',
                        type=argparse.FileType('w'), default='out.mp3')
    parser.add_argument('-l', '--language', action='store',
                        nargs='?', help='Language to output text to.', default='en')
    group = parser.add_mutually_exclusive_group(required=True)
    group.add_argument(
        '-f', '--file', type=argparse.FileType('r'), help='File to read text from.')
    group.add_argument('-s', '--string', action='store',
                       nargs='+', help='A string of text to convert to speech.')

    if len(sys.argv) == 1:
        parser.print_help()
        sys.exit(1)

    args = parser.parse_args()
    if args.file:
        text = args.file.read()
    if args.string:
        text = ' '.join(map(str, args.string))

    # process text into chunks
    google_speak(args.language, text, args.output)
    args.output.close()

    print('Saved MP3 to %s' % args.output.name)

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before># Copyright 2020 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

""" Tests for the pr_stats module. """

import csv
from datetime import datetime
import os
import sys
import unittest
from unittest.mock import patch
import pr_stats

class PrStatsTest(unittest.TestCase):
    """ Repos test class. """

    def test_get_comments_no_file(self):
        """ Test to check if unsupported arguments will raise exception. """
        sys.argv = ["repos.py", "repo_type"]
        self.assertFalse(os.path.isfile("data/repo_type.csv"))
        with self.assertRaises(Exception):
            pr_stats.main()

    def test_calculate_week(self):
        """ Test to check that weeks are calculated correctly. """
        start_date1 = "05/12/2020"
        created_date1 = datetime.strptime("05/12/2020", "%m/%d/%Y")
        week1 = pr_stats.calculate_week(start_date1, created_date1)
        self.assertEqual(week1, 1)

        start_date2 = "05/12/2020"
        created_date2 = datetime.strptime("05/18/2020", "%m/%d/%Y")
        week2 = pr_stats.calculate_week(start_date2, created_date2)
        self.assertEqual(week2, 1)

        start_date3 = "05/12/2020"
        created_date3 = datetime.strptime("05/19/2020", "%m/%d/%Y")
        week3 = pr_stats.calculate_week(start_date3, created_date3)
        self.assertEqual(week3, 2)

    def test_get_start_date(self):
        """ Test to check if start dates are found correctly based on
            participants. """
        participants = [
            {
                "login": "user1"
            },
            {
                "login": "user2"
            },
            {
                "login": "user3"
            }
        ]

        host_dict = dict()
        repo_dates = dict()
        repo = "test_repo"
        start_date = pr_stats.get_start_date(
            participants, host_dict, repo_dates, repo)
        self.assertEqual(start_date, "unknown")

        host_dict["user3"] = "05/12/2020"
        start_date = pr_stats.get_start_date(
            participants, host_dict, repo_dates, repo)
        self.assertEqual(start_date, "05/12/2020")
        self.assertEqual(repo_dates[repo], "05/12/2020")

    @patch("pr_stats.get_pr_stats")
    def test_get_pr_stats(self, mock_results):
        """ Test for getting test repository PR statistics.

        This test uses a CSV with test repositories and checks if the
        output pull request statistics CSV file is created correctly.
        The generated test files are deleted upon test completion.
        """

        # Mock results: pull requests for RISR repository.
        # pylint: disable=line-too-long
        mock_results.return_value = {
            "data": {
                "repository": {
                    "nameWithOwner": "googleinterns/risr",
                    "pullRequests": {
                        "nodes": [{
                            "resourcePath": "/googleinterns/risr/pull/1",
                            "number": 1,
                            "createdAt": "2020-06-11T20:54:31Z",
                            "closedAt": "2020-06-12T19:36:08Z",
                            "deletions": 0,
                            "additions": 653,
                            "comments": {
                                "totalCount": 0
                            },
                            "reviews": {
                                "nodes": [{
                                    "body": "text1",
                                    "comments": {
                                        "totalCount": 3
                                    }
                                }, {
                                    "body": "",
                                    "comments": {
                                        "totalCount": 1
                                    }
                                }, {
                                    "body":
                                    "text2",
                                    "comments": {
                                        "totalCount": 2
                                    }
                                }, {
                                    "body": "text3",
                                    "comments": {
                                        "totalCount": 2
                                    }
                                }]
                            },
                            "participants": {
                                "nodes": [
                                    {"host1": "05/12/2020"}
                                ]
                            },
                            "timelineItems": {
                                "nodes": [
                                    {
                                        "state": "CHANGES_REQUESTED"
                                    },
                                    {},
                                    {
                                        "state": "APPROVED"
                                    }
                                ]
                            }
                        }, {
                            "resourcePath": "/googleinterns/risr/pull/2",
                            "number": 2,
                            "createdAt": "2020-06-12T22:01:36Z",
                            "closedAt": "2020-06-24T16:30:31Z",
                            "deletions": 48,
                            "additions": 631,
                            "comments": {
                                "totalCount": 0
                            },
                            "reviews": {
                                "nodes": [{
                                    "body": "",
                                    "comments": {
                                        "totalCount": 1
                                    }
                                }, {
                                    "body":
                                    "text4",
                                    "comments": {
                                        "totalCount": 0
                                    }
                                }]
                            },
                            "participants": {
                                "nodes": [
                                    {"host1": "05/12/2020"}
                                ]
                            },
                            "timelineItems": {
                                "nodes": [
                                    {
                                        "state": "CHANGES_REQUESTED"
                                    },
                                    {},
                                    {
                                        "state": "APPROVED"
                                    }
                                ]
                            }
                        }, {
                            "resourcePath": "/googleinterns/risr/pull/3",
                            "number": 3,
                            "createdAt": "2020-06-24T16:36:50Z",
                            "closedAt": "2020-06-24T20:25:02Z",
                            "deletions": 19,
                            "additions": 61,
                            "comments": {
                                "totalCount": 1
                            },
                            "reviews": {
                                "nodes": [{
                                    "body": "text5",
                                    "comments": {
                                        "totalCount": 2
                                    }
                                }, {
                                    "body": "",
                                    "comments": {
                                        "totalCount": 1
                                    }
                                }, {
                                    "body": "",
                                    "comments": {
                                        "totalCount": 1
                                    }
                                }, {
                                    "body": "",
                                    "comments": {
                                        "totalCount": 0
                                    }
                                }]
                            },
                            "participants": {
                                "nodes": [
                                    {"host1": "05/12/2020"}
                                ]
                            },
                            "timelineItems": {
                                "nodes": [
                                    {
                                        "state": "CHANGES_REQUESTED"
                                    },
                                    {},
                                    {
                                        "state": "APPROVED"
                                    }
                                ]
                            }
                        }, {
                            "resourcePath": "/googleinterns/risr/pull/4",
                            "number": 4,
                            "createdAt": "2020-06-26T22:26:47Z",
                            "closedAt": None,
                            "deletions": 32,
                            "additions": 392,
                            "comments": {
                                "totalCount": 0
                            },
                            "reviews": {
                                "nodes": []
                            },
                            "participants": {
                                "nodes": [
                                    {"host1": "05/12/2020"}
                                ]
                            },
                            "timelineItems": {
                                "nodes": [
                                    {
                                        "state": "CHANGES_REQUESTED"
                                    },
                                    {},
                                    {
                                        "state": "APPROVED"
                                    }
                                ]
                            }
                        }]
                    }
                }
            }
        }

        pr_stats_path = "data/test_pr_stats.csv"

        if os.path.isfile(pr_stats_path):
            os.remove(pr_stats_path)
        self.assertFalse(os.path.isfile(pr_stats_path))

        sys.argv = ["repos.py", "test"]
        pr_stats.main()
        self.assertTrue(os.path.isfile(pr_stats_path))
        with open(pr_stats_path) as in_csv:
            reader = csv.DictReader(in_csv)
            self.assertGreater(len(list(reader)), 1)
            for i, row in enumerate(reader, 1):
                self.assertTrue("/googleinterns/risr/pull/" in row["pr_path"])
                self.assertEqual(row["pr_number"], i)
                self.assertEqual(row["start_date"], "05/12/2020")
        os.remove(pr_stats_path)

if __name__ == "__main__":
    unittest.main()
<commit_msg>Test review count.<commit_after># Copyright 2020 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

""" Tests for the pr_stats module. """

import csv
from datetime import datetime
import os
import sys
import unittest
from unittest.mock import patch
import pr_stats

class PrStatsTest(unittest.TestCase):
    """ Repos test class. """

    def test_get_comments_no_file(self):
        """ Test to check if unsupported arguments will raise exception. """
        sys.argv = ["repos.py", "repo_type"]
        self.assertFalse(os.path.isfile("data/repo_type.csv"))
        with self.assertRaises(Exception):
            pr_stats.main()

    def test_calculate_week(self):
        """ Test to check that weeks are calculated correctly. """
        start_date1 = "05/12/2020"
        created_date1 = datetime.strptime("05/12/2020", "%m/%d/%Y")
        week1 = pr_stats.calculate_week(start_date1, created_date1)
        self.assertEqual(week1, 1)

        start_date2 = "05/12/2020"
        created_date2 = datetime.strptime("05/18/2020", "%m/%d/%Y")
        week2 = pr_stats.calculate_week(start_date2, created_date2)
        self.assertEqual(week2, 1)

        start_date3 = "05/12/2020"
        created_date3 = datetime.strptime("05/19/2020", "%m/%d/%Y")
        week3 = pr_stats.calculate_week(start_date3, created_date3)
        self.assertEqual(week3, 2)

    def test_get_start_date(self):
        """ Test to check if start dates are found correctly based on
            participants. """
        participants = [
            {
                "login": "user1"
            },
            {
                "login": "user2"
            },
            {
                "login": "user3"
            }
        ]

        host_dict = dict()
        repo_dates = dict()
        repo = "test_repo"
        start_date = pr_stats.get_start_date(
            participants, host_dict, repo_dates, repo)
        self.assertEqual(start_date, "unknown")

        host_dict["user3"] = "05/12/2020"
        start_date = pr_stats.get_start_date(
            participants, host_dict, repo_dates, repo)
        self.assertEqual(start_date, "05/12/2020")
        self.assertEqual(repo_dates[repo], "05/12/2020")

    @patch("pr_stats.get_pr_stats")
    def test_get_pr_stats(self, mock_results):
        """ Test for getting test repository PR statistics.

        This test uses a CSV with test repositories and checks if the
        output pull request statistics CSV file is created correctly.
        The generated test files are deleted upon test completion.
        """

        # Mock results: pull requests for RISR repository.
        # pylint: disable=line-too-long
        mock_results.return_value = {
            "data": {
                "repository": {
                    "nameWithOwner": "googleinterns/risr",
                    "pullRequests": {
                        "nodes": [{
                            "resourcePath": "/googleinterns/risr/pull/1",
                            "number": 1,
                            "createdAt": "2020-06-11T20:54:31Z",
                            "closedAt": "2020-06-12T19:36:08Z",
                            "deletions": 0,
                            "additions": 653,
                            "comments": {
                                "totalCount": 0
                            },
                            "reviews": {
                                "nodes": [{
                                    "body": "text1",
                                    "comments": {
                                        "totalCount": 3
                                    }
                                }, {
                                    "body": "",
                                    "comments": {
                                        "totalCount": 1
                                    }
                                }, {
                                    "body":
                                    "text2",
                                    "comments": {
                                        "totalCount": 2
                                    }
                                }, {
                                    "body": "text3",
                                    "comments": {
                                        "totalCount": 2
                                    }
                                }]
                            },
                            "participants": {
                                "nodes": [
                                    {"host1": "05/12/2020"}
                                ]
                            },
                            "timelineItems": {
                                "nodes": [
                                    {
                                        "state": "CHANGES_REQUESTED"
                                    },
                                    {},
                                    {
                                        "state": "APPROVED"
                                    }
                                ]
                            }
                        }, {
                            "resourcePath": "/googleinterns/risr/pull/2",
                            "number": 2,
                            "createdAt": "2020-06-12T22:01:36Z",
                            "closedAt": "2020-06-24T16:30:31Z",
                            "deletions": 48,
                            "additions": 631,
                            "comments": {
                                "totalCount": 0
                            },
                            "reviews": {
                                "nodes": [{
                                    "body": "",
                                    "comments": {
                                        "totalCount": 1
                                    }
                                }, {
                                    "body":
                                    "text4",
                                    "comments": {
                                        "totalCount": 0
                                    }
                                }]
                            },
                            "participants": {
                                "nodes": [
                                    {"host1": "05/12/2020"}
                                ]
                            },
                            "timelineItems": {
                                "nodes": [
                                    {
                                        "state": "CHANGES_REQUESTED"
                                    },
                                    {},
                                    {
                                        "state": "APPROVED"
                                    }
                                ]
                            }
                        }, {
                            "resourcePath": "/googleinterns/risr/pull/3",
                            "number": 3,
                            "createdAt": "2020-06-24T16:36:50Z",
                            "closedAt": "2020-06-24T20:25:02Z",
                            "deletions": 19,
                            "additions": 61,
                            "comments": {
                                "totalCount": 1
                            },
                            "reviews": {
                                "nodes": [{
                                    "body": "text5",
                                    "comments": {
                                        "totalCount": 2
                                    }
                                }, {
                                    "body": "",
                                    "comments": {
                                        "totalCount": 1
                                    }
                                }, {
                                    "body": "",
                                    "comments": {
                                        "totalCount": 1
                                    }
                                }, {
                                    "body": "",
                                    "comments": {
                                        "totalCount": 0
                                    }
                                }]
                            },
                            "participants": {
                                "nodes": [
                                    {"host1": "05/12/2020"}
                                ]
                            },
                            "timelineItems": {
                                "nodes": [
                                    {
                                        "state": "CHANGES_REQUESTED"
                                    },
                                    {},
                                    {
                                        "state": "APPROVED"
                                    }
                                ]
                            }
                        }, {
                            "resourcePath": "/googleinterns/risr/pull/4",
                            "number": 4,
                            "createdAt": "2020-06-26T22:26:47Z",
                            "closedAt": None,
                            "deletions": 32,
                            "additions": 392,
                            "comments": {
                                "totalCount": 0
                            },
                            "reviews": {
                                "nodes": []
                            },
                            "participants": {
                                "nodes": [
                                    {"host1": "05/12/2020"}
                                ]
                            },
                            "timelineItems": {
                                "nodes": [
                                    {
                                        "state": "CHANGES_REQUESTED"
                                    },
                                    {},
                                    {
                                        "state": "APPROVED"
                                    }
                                ]
                            }
                        }]
                    }
                }
            }
        }

        pr_stats_path = "data/test_pr_stats.csv"

        if os.path.isfile(pr_stats_path):
            os.remove(pr_stats_path)
        self.assertFalse(os.path.isfile(pr_stats_path))

        sys.argv = ["repos.py", "test"]
        pr_stats.main()
        self.assertTrue(os.path.isfile(pr_stats_path))
        with open(pr_stats_path) as in_csv:
            reader = csv.DictReader(in_csv)
            self.assertGreater(len(list(reader)), 1)
            for i, row in enumerate(reader, 1):
                self.assertTrue("/googleinterns/risr/pull/" in row["pr_path"])
                self.assertEqual(row["pr_number"], i)
                self.assertEqual(row["start_date"], "05/12/2020")
                self.assertEqual(row["review_count"], 2)
        os.remove(pr_stats_path)

if __name__ == "__main__":
    unittest.main()
<|endoftext|>


<commit_before># Copyright 2020 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import json
from yaml import dump, load, Dumper, Loader

from data_validation import cli_tools, consts
from data_validation.config_manager import ConfigManager
from data_validation.data_validation import DataValidation

def _get_arg_config_file(args):
    """Return String yaml config file path."""
    if not args.config_file:
        raise ValueError("YAML Config File was not supplied.")

    return args.config_file

def _get_yaml_config_from_file(config_file_path):
    """Return Dict of yaml validation data."""
    with open(config_file_path, "r") as yaml_file:
        yaml_configs = load(yaml_file.read(), Loader=Loader)

    return yaml_configs

def get_aggregate_config(args, config_manager):
    """Return list of formated aggregation objects.

    Args:
        config_manager (ConfigManager): Validation config manager instance.
    """
    aggregate_configs = [config_manager.build_config_count_aggregate()]

    if args.count:
        col_args = None if args.count == "*" else json.loads(args.count)
        aggregate_configs += config_manager.build_config_column_aggregates(
            "count", col_args, None
        )
    if args.sum:
        col_args = None if args.sum == "*" else json.loads(args.sum)
        aggregate_configs += config_manager.build_config_column_aggregates(
            "sum", col_args, ["int64", "float64"]
        )

    return aggregate_configs

def build_config_from_args(args, config_manager):
    """Return config manager object ready to execute.

    Args:
        config_manager (ConfigManager): Validation config manager instance.
    """
    config_manager.append_aggregates(get_aggregate_config(args, config_manager))
    if config_manager.validation_type == "GroupedColumn":
        grouped_columns = json.loads(args.grouped_columns)
        config_manager.append_query_groups(
            config_manager.build_config_grouped_columns(grouped_columns)
        )
    # TODO(GH#18): Add query filter config logic

    return config_manager

def build_config_managers_from_args(args):
    """Return a list of config managers ready to execute."""
    configs = []

    config_type = args.type
    source_conn = cli_tools.get_connection(args.source_conn)
    target_conn = cli_tools.get_connection(args.target_conn)

    result_handler_config = None
    if args.result_handler_config:
        result_handler_config = json.loads(args.result_handler_config)

    source_client = DataValidation.get_data_client(source_conn)
    target_client = DataValidation.get_data_client(target_conn)

    tables_list = json.loads(args.tables_list)
    for table_obj in tables_list:
        config_manager = ConfigManager.build_config_manager(
            config_type,
            source_conn,
            target_conn,
            source_client,
            target_client,
            table_obj,
            result_handler_config=result_handler_config,
            verbose=args.verbose,
        )
        configs.append(build_config_from_args(args, config_manager))

    return configs

def build_config_managers_from_yaml(args):
    """Returns List[ConfigManager] instances ready to be executed."""
    config_managers = []

    config_file_path = _get_arg_config_file(args)
    yaml_configs = _get_yaml_config_from_file(config_file_path)

    source_conn = cli_tools.get_connection(yaml_configs[consts.YAML_SOURCE])
    target_conn = cli_tools.get_connection(yaml_configs[consts.YAML_TARGET])

    source_client = DataValidation.get_data_client(source_conn)
    target_client = DataValidation.get_data_client(target_conn)

    for config in yaml_configs[consts.YAML_VALIDATIONS]:
        config[consts.CONFIG_SOURCE_CONN] = source_conn
        config[consts.CONFIG_TARGET_CONN] = target_conn
        config[consts.CONFIG_RESULT_HANDLER] = yaml_configs[consts.YAML_RESULT_HANDLER]
        config_manager = ConfigManager(
            config, source_client, target_client, verbose=args.verbose
        )

        config_managers.append(config_manager)

    return config_managers

def convert_config_to_yaml(args, config_managers):
    """Return dict objects formatted for yaml validations.

    Args:
        config_managers (list[ConfigManager]): List of config manager instances.
    """
    yaml_config = {
        consts.YAML_SOURCE: args.source_conn,
        consts.YAML_TARGET: args.target_conn,
        consts.YAML_RESULT_HANDLER: config_managers[0].result_handler_config,
        consts.YAML_VALIDATIONS: [],
    }

    for config_manager in config_managers:
        yaml_config[consts.YAML_VALIDATIONS].append(
            config_manager.get_yaml_validation_block()
        )

    return yaml_config

def run_validation(config_manager, verbose=False):
    """Run a single validation.

    Args:
        config_manager (ConfigManager): Validation config manager instance.
        verbose (bool): Validation setting to log queries run.
    """
    validator = DataValidation(
        config_manager.config,
        validation_builder=None,
        result_handler=None,
        verbose=verbose,
    )
    validator.execute()

def run_validations(args, config_managers):
    """Run and manage a series of validations.

    Args:
        config_managers (list[ConfigManager]): List of config manager instances.
    """
    # TODO(issue/31): Add parallel execution logic
    for config_manager in config_managers:
        run_validation(config_manager, verbose=args.verbose)

def store_yaml_config_file(args, config_managers):
    """Build a YAML config file fromt he supplied configs.

    Args:
        config_managers (list[ConfigManager]): List of config manager instances.
    """
    config_file_path = _get_arg_config_file(args)
    yaml_configs = convert_config_to_yaml(args, config_managers)
    yaml_config_str = dump(yaml_configs, Dumper=Dumper)

    with open(config_file_path, "w") as yaml_file:
        yaml_file.write(yaml_config_str)

def run(args):
    """ """
    config_managers = build_config_managers_from_args(args)

    if args.config_file:
        store_yaml_config_file(args, config_managers)
    else:
        run_validations(args, config_managers)

def run_connections(args):
    """ Run commands related to connection management."""
    if args.connect_cmd == "list":
        cli_tools.list_connections()
    elif args.connect_cmd == "add":
        conn = cli_tools.get_connection_config_from_args(args)
        DataValidation.get_data_client(conn)
        cli_tools.store_connection(args.connection_name, conn)
    else:
        raise ValueError(f"Connections Argument '{args.connect_cmd}' is not supported")

def main():
    # Create Parser and Get Deployment Info
    args = cli_tools.get_parsed_args()

    if args.command == "run":
        run(args)
    elif args.command == "connections":
        run_connections(args)
    elif args.command == "run-config":
        config_managers = build_config_managers_from_yaml(args)
        run_validations(args, config_managers)
    else:
        raise ValueError(f"Positional Argument '{args.command}' is not supported")

if __name__ == "__main__":
    main()
<commit_msg>add comment for why we get but dont use a connection<commit_after># Copyright 2020 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import json
from yaml import dump, load, Dumper, Loader

from data_validation import cli_tools, consts
from data_validation.config_manager import ConfigManager
from data_validation.data_validation import DataValidation

def _get_arg_config_file(args):
    """Return String yaml config file path."""
    if not args.config_file:
        raise ValueError("YAML Config File was not supplied.")

    return args.config_file

def _get_yaml_config_from_file(config_file_path):
    """Return Dict of yaml validation data."""
    with open(config_file_path, "r") as yaml_file:
        yaml_configs = load(yaml_file.read(), Loader=Loader)

    return yaml_configs

def get_aggregate_config(args, config_manager):
    """Return list of formated aggregation objects.

    Args:
        config_manager (ConfigManager): Validation config manager instance.
    """
    aggregate_configs = [config_manager.build_config_count_aggregate()]

    if args.count:
        col_args = None if args.count == "*" else json.loads(args.count)
        aggregate_configs += config_manager.build_config_column_aggregates(
            "count", col_args, None
        )
    if args.sum:
        col_args = None if args.sum == "*" else json.loads(args.sum)
        aggregate_configs += config_manager.build_config_column_aggregates(
            "sum", col_args, ["int64", "float64"]
        )

    return aggregate_configs

def build_config_from_args(args, config_manager):
    """Return config manager object ready to execute.

    Args:
        config_manager (ConfigManager): Validation config manager instance.
    """
    config_manager.append_aggregates(get_aggregate_config(args, config_manager))
    if config_manager.validation_type == "GroupedColumn":
        grouped_columns = json.loads(args.grouped_columns)
        config_manager.append_query_groups(
            config_manager.build_config_grouped_columns(grouped_columns)
        )
    # TODO(GH#18): Add query filter config logic

    return config_manager

def build_config_managers_from_args(args):
    """Return a list of config managers ready to execute."""
    configs = []

    config_type = args.type
    source_conn = cli_tools.get_connection(args.source_conn)
    target_conn = cli_tools.get_connection(args.target_conn)

    result_handler_config = None
    if args.result_handler_config:
        result_handler_config = json.loads(args.result_handler_config)

    source_client = DataValidation.get_data_client(source_conn)
    target_client = DataValidation.get_data_client(target_conn)

    tables_list = json.loads(args.tables_list)
    for table_obj in tables_list:
        config_manager = ConfigManager.build_config_manager(
            config_type,
            source_conn,
            target_conn,
            source_client,
            target_client,
            table_obj,
            result_handler_config=result_handler_config,
            verbose=args.verbose,
        )
        configs.append(build_config_from_args(args, config_manager))

    return configs

def build_config_managers_from_yaml(args):
    """Returns List[ConfigManager] instances ready to be executed."""
    config_managers = []

    config_file_path = _get_arg_config_file(args)
    yaml_configs = _get_yaml_config_from_file(config_file_path)

    source_conn = cli_tools.get_connection(yaml_configs[consts.YAML_SOURCE])
    target_conn = cli_tools.get_connection(yaml_configs[consts.YAML_TARGET])

    source_client = DataValidation.get_data_client(source_conn)
    target_client = DataValidation.get_data_client(target_conn)

    for config in yaml_configs[consts.YAML_VALIDATIONS]:
        config[consts.CONFIG_SOURCE_CONN] = source_conn
        config[consts.CONFIG_TARGET_CONN] = target_conn
        config[consts.CONFIG_RESULT_HANDLER] = yaml_configs[consts.YAML_RESULT_HANDLER]
        config_manager = ConfigManager(
            config, source_client, target_client, verbose=args.verbose
        )

        config_managers.append(config_manager)

    return config_managers

def convert_config_to_yaml(args, config_managers):
    """Return dict objects formatted for yaml validations.

    Args:
        config_managers (list[ConfigManager]): List of config manager instances.
    """
    yaml_config = {
        consts.YAML_SOURCE: args.source_conn,
        consts.YAML_TARGET: args.target_conn,
        consts.YAML_RESULT_HANDLER: config_managers[0].result_handler_config,
        consts.YAML_VALIDATIONS: [],
    }

    for config_manager in config_managers:
        yaml_config[consts.YAML_VALIDATIONS].append(
            config_manager.get_yaml_validation_block()
        )

    return yaml_config

def run_validation(config_manager, verbose=False):
    """Run a single validation.

    Args:
        config_manager (ConfigManager): Validation config manager instance.
        verbose (bool): Validation setting to log queries run.
    """
    validator = DataValidation(
        config_manager.config,
        validation_builder=None,
        result_handler=None,
        verbose=verbose,
    )
    validator.execute()

def run_validations(args, config_managers):
    """Run and manage a series of validations.

    Args:
        config_managers (list[ConfigManager]): List of config manager instances.
    """
    # TODO(issue/31): Add parallel execution logic
    for config_manager in config_managers:
        run_validation(config_manager, verbose=args.verbose)

def store_yaml_config_file(args, config_managers):
    """Build a YAML config file fromt he supplied configs.

    Args:
        config_managers (list[ConfigManager]): List of config manager instances.
    """
    config_file_path = _get_arg_config_file(args)
    yaml_configs = convert_config_to_yaml(args, config_managers)
    yaml_config_str = dump(yaml_configs, Dumper=Dumper)

    with open(config_file_path, "w") as yaml_file:
        yaml_file.write(yaml_config_str)

def run(args):
    """ """
    config_managers = build_config_managers_from_args(args)

    if args.config_file:
        store_yaml_config_file(args, config_managers)
    else:
        run_validations(args, config_managers)

def run_connections(args):
    """ Run commands related to connection management."""
    if args.connect_cmd == "list":
        cli_tools.list_connections()
    elif args.connect_cmd == "add":
        conn = cli_tools.get_connection_config_from_args(args)
        # Test getting a client to validate connection details
        _ = DataValidation.get_data_client(conn)
        cli_tools.store_connection(args.connection_name, conn)
    else:
        raise ValueError(f"Connections Argument '{args.connect_cmd}' is not supported")

def main():
    # Create Parser and Get Deployment Info
    args = cli_tools.get_parsed_args()

    if args.command == "run":
        run(args)
    elif args.command == "connections":
        run_connections(args)
    elif args.command == "run-config":
        config_managers = build_config_managers_from_yaml(args)
        run_validations(args, config_managers)
    else:
        raise ValueError(f"Positional Argument '{args.command}' is not supported")

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before>from flask import request
from dataactbroker.handlers.accountHandler import AccountHandler
from dataactbroker.permissions import permissions_check
from dataactbroker.routeUtils import RouteUtils

def add_user_routes(app,system_email,bcrypt):
    """ Create routes related to file submission for flask app

    """

    RouteUtils.SYSTEM_EMAIL = system_email # Set the system email to be used

    @app.route("/v1/register/", methods = ["POST"])
    #check the session to make sure register is set to prevent any one from using route
    @permissions_check(permissionList=["check_email_token"])
    def register_user():
        """ Expects request to have keys 'email', 'name', 'agency', and 'title' """
        accountManager = AccountHandler(request,bcrypt = bcrypt)
        return RouteUtils.run_instance_function(accountManager,accountManager.register, getSystemEmail = True, getSession = True)

    @app.route("/v1/change_status/", methods = ["POST"])
    @permissions_check(permissionList=["website_admin"])
    def change_status():
        """ Expects request to have keys 'user_email' and 'new_status' """
        accountManager = AccountHandler(request,bcrypt = bcrypt)
        return RouteUtils.run_instance_function(accountManager, accountManager.changeStatus,getSystemEmail = True)

    @app.route("/v1/confirm_email/", methods = ["POST"])
    def confirm():
        """ Expects request to have email  """
        accountManager = AccountHandler(request,bcrypt = bcrypt)
        return RouteUtils.run_instance_function(accountManager, accountManager.createEmailConfirmation, True,True)

    @app.route("/v1/confirm_email_token/", methods = ["POST"])
    def checkEmailToken():
        """ Expects request to have email  """
        accountManager = AccountHandler(request,bcrypt = bcrypt)
        return RouteUtils.run_instance_function(accountManager, accountManager.checkEmailConfirmationToken, getSession = True)

    @app.route("/v1/confirm_password_token/", methods = ["POST"])
    def checkPasswordToken():
        """ Expects request to have email  """
        accountManager = AccountHandler(request,bcrypt = bcrypt)
        return RouteUtils.run_instance_function(accountManager, accountManager.checkPasswordToken, getSession = True)

    @app.route("/v1/list_users_with_status/", methods = ["POST"])
    @permissions_check(permissionList=["website_admin"])
    def list_users_with_status():
        """ Expects request to have key 'status', will list all users with that status """
        accountManager = AccountHandler(request,bcrypt = bcrypt)
        return RouteUtils.run_instance_function(accountManager, accountManager.listUsersWithStatus)

    @app.route("/v1/list_submissions/", methods = ["GET"])
    @permissions_check
    def list_submissions():
        """ List submission IDs associated with the current user """
        accountManager = AccountHandler(request,bcrypt = bcrypt)
        return RouteUtils.run_instance_function(accountManager, accountManager.listSubmissionsByCurrentUser)

    @app.route("/v1/set_password/", methods=["POST"])
    @permissions_check(permissionList=["check_password_token"])
    def set_password():
        """ Set a new password for specified user """
        accountManager = AccountHandler(request,bcrypt = bcrypt)
        return RouteUtils.run_instance_function(accountManager, accountManager.setNewPassword)

    @app.route("/v1/reset_password/", methods=["POST"])
    def reset_password():
        """ Removes current password from DB and sends email with token for user to reset their password.  Expects 'email' key in request body. """
        accountManager = AccountHandler(request,bcrypt = bcrypt)
        return RouteUtils.run_instance_function(accountManager, accountManager.resetPassword, True,True)

    @app.route("/v1/current_user/", methods=["GET"])
    @permissions_check
    def current_user():
        """ gets the current user information """
        accountManager = AccountHandler(request,bcrypt = bcrypt)
        return RouteUtils.run_instance_function(accountManager, accountManager.getCurrentUser, getSession = True)
<commit_msg>Minor comment fix to checkEmailToken route<commit_after>from flask import request
from dataactbroker.handlers.accountHandler import AccountHandler
from dataactbroker.permissions import permissions_check
from dataactbroker.routeUtils import RouteUtils

def add_user_routes(app,system_email,bcrypt):
    """ Create routes related to file submission for flask app

    """

    RouteUtils.SYSTEM_EMAIL = system_email # Set the system email to be used

    @app.route("/v1/register/", methods = ["POST"])
    #check the session to make sure register is set to prevent any one from using route
    @permissions_check(permissionList=["check_email_token"])
    def register_user():
        """ Expects request to have keys 'email', 'name', 'agency', and 'title' """
        accountManager = AccountHandler(request,bcrypt = bcrypt)
        return RouteUtils.run_instance_function(accountManager,accountManager.register, getSystemEmail = True, getSession = True)

    @app.route("/v1/change_status/", methods = ["POST"])
    @permissions_check(permissionList=["website_admin"])
    def change_status():
        """ Expects request to have keys 'user_email' and 'new_status' """
        accountManager = AccountHandler(request,bcrypt = bcrypt)
        return RouteUtils.run_instance_function(accountManager, accountManager.changeStatus,getSystemEmail = True)

    @app.route("/v1/confirm_email/", methods = ["POST"])
    def confirm():
        """ Expects request to have email  """
        accountManager = AccountHandler(request,bcrypt = bcrypt)
        return RouteUtils.run_instance_function(accountManager, accountManager.createEmailConfirmation, True,True)

    @app.route("/v1/confirm_email_token/", methods = ["POST"])
    def checkEmailToken():
        """ Expects request to have token  """
        accountManager = AccountHandler(request,bcrypt = bcrypt)
        return RouteUtils.run_instance_function(accountManager, accountManager.checkEmailConfirmationToken, getSession = True)

    @app.route("/v1/confirm_password_token/", methods = ["POST"])
    def checkPasswordToken():
        """ Expects request to have email  """
        accountManager = AccountHandler(request,bcrypt = bcrypt)
        return RouteUtils.run_instance_function(accountManager, accountManager.checkPasswordToken, getSession = True)

    @app.route("/v1/list_users_with_status/", methods = ["POST"])
    @permissions_check(permissionList=["website_admin"])
    def list_users_with_status():
        """ Expects request to have key 'status', will list all users with that status """
        accountManager = AccountHandler(request,bcrypt = bcrypt)
        return RouteUtils.run_instance_function(accountManager, accountManager.listUsersWithStatus)

    @app.route("/v1/list_submissions/", methods = ["GET"])
    @permissions_check
    def list_submissions():
        """ List submission IDs associated with the current user """
        accountManager = AccountHandler(request,bcrypt = bcrypt)
        return RouteUtils.run_instance_function(accountManager, accountManager.listSubmissionsByCurrentUser)

    @app.route("/v1/set_password/", methods=["POST"])
    @permissions_check(permissionList=["check_password_token"])
    def set_password():
        """ Set a new password for specified user """
        accountManager = AccountHandler(request,bcrypt = bcrypt)
        return RouteUtils.run_instance_function(accountManager, accountManager.setNewPassword)

    @app.route("/v1/reset_password/", methods=["POST"])
    def reset_password():
        """ Removes current password from DB and sends email with token for user to reset their password.  Expects 'email' key in request body. """
        accountManager = AccountHandler(request,bcrypt = bcrypt)
        return RouteUtils.run_instance_function(accountManager, accountManager.resetPassword, True,True)

    @app.route("/v1/current_user/", methods=["GET"])
    @permissions_check
    def current_user():
        """ gets the current user information """
        accountManager = AccountHandler(request,bcrypt = bcrypt)
        return RouteUtils.run_instance_function(accountManager, accountManager.getCurrentUser, getSession = True)
<|endoftext|>