<commit_before># -*- coding: utf-8 -*-
import os.path

from intelmq.lib.bot import CollectorBot

try:
    import stomp
except ImportError:
    stomp = None
else:
    class StompListener(stomp.listener.PrintingListener):
        """
        the stomp listener gets called asynchronously for
        every STOMP message
        """
        def __init__(self, n6stompcollector):
            self.stompbot = n6stompcollector

        def on_heartbeat_timeout(self):
            self.stompbot.logger.info("Heartbeat timeout. Attempting to re-connect.")
            self.connect_and_subscribe()

        def connect_and_subscribe(self):
            self.conn.start()
            connect_status = self.conn.connect(wait=True)
            subscribe_status = self.conn.subscribe(destination=self.destination,
                                                   id=1, ack='auto')
            self.stompbot.logger.info('Successfully connected and subscribed. '
                                      'Connect status: %r, subscribe status: %r.',
                                      connect_status, subscribe_status)

        def on_error(self, headers, message):
            self.stompbot.logger.error('Received an error: %r.', message)

        def on_message(self, headers, message):
            self.stompbot.logger.debug('Receive message %r...', message[:500])
            report = self.stompbot.new_report()
            report.add("raw", message.rstrip())
            report.add("feed.url", "stomp://" +
                       self.stompbot.parameters.server +
                       ":" + str(self.stompbot.parameters.port) +
                       "/" + self.stompbot.parameters.exchange)
            self.stompbot.send_message(report)

        def on_disconnected(self):
            self.stompbot.logger.debug('Detected disconnect')
            self.connect_and_subscribe()

class StompCollectorBot(CollectorBot):
    """ main class for the STOMP protocol collector """

    collector_empty_process = True

    def init(self):
        if stomp is None:
            raise ValueError('Could not import stomp. Please install it.')

        self.server = getattr(self.parameters, 'server', 'n6stream.cert.pl')
        self.port = getattr(self.parameters, 'port', 61614)
        self.exchange = getattr(self.parameters, 'exchange', '')
        self.heartbeat = getattr(self.parameters, 'heartbeat', 60000)
        self.ssl_ca_cert = getattr(self.parameters, 'ssl_ca_certificate',
                                   'ca.pem')
        self.ssl_cl_cert = getattr(self.parameters, 'ssl_client_certificate',
                                   'client.pem')
        self.ssl_cl_cert_key = getattr(self.parameters,
                                       'ssl_client_certificate_key',
                                       'client.key')
        self.http_verify_cert = getattr(self.parameters,
                                        'http_verify_cert', True)

        # check if certificates exist
        for f in [self.ssl_ca_cert, self.ssl_cl_cert, self.ssl_cl_cert_key]:
            if not os.path.isfile(f):
                raise ValueError("Could not open file %r." % f)

        _host = [(self.server, self.port)]
        self.conn = stomp.Connection(host_and_ports=_host, use_ssl=True,
                                     ssl_key_file=self.ssl_cl_cert_key,
                                     ssl_cert_file=self.ssl_cl_cert,
                                     ssl_ca_certs=self.ssl_ca_cert,
                                     wait_on_receipt=True,
                                     heartbeats=(self.heartbeat,
                                                 self.heartbeat))

        self.conn.set_listener('', StompListener(self))
        self.conn.start()
        self.conn.destination = self.exchange
        self.conn.connect_and_subscribe()

    def shutdown(self):
        try:
            self.conn.disconnect()
        except stomp.exception.NotConnectedException:
            pass

    def process(self):
        pass

BOT = StompCollectorBot
<commit_msg>BUG: stomp collector: fix connection on startup<commit_after># -*- coding: utf-8 -*-
import os.path

from intelmq.lib.bot import CollectorBot

try:
    import stomp
except ImportError:
    stomp = None
else:
    class StompListener(stomp.listener.PrintingListener):
        """
        the stomp listener gets called asynchronously for
        every STOMP message
        """
        def __init__(self, n6stompcollector, conn, destination):
            self.stompbot = n6stompcollector
            self.conn = conn
            self.destination = destination

        def on_heartbeat_timeout(self):
            self.stompbot.logger.info("Heartbeat timeout. Attempting to re-connect.")
            connect_and_subscribe(self.conn, self.stompbot.logger, self.destination)

        def on_error(self, headers, message):
            self.stompbot.logger.error('Received an error: %r.', message)

        def on_message(self, headers, message):
            self.stompbot.logger.debug('Receive message %r...', message[:500])
            report = self.stompbot.new_report()
            report.add("raw", message.rstrip())
            report.add("feed.url", "stomp://" +
                       self.stompbot.parameters.server +
                       ":" + str(self.stompbot.parameters.port) +
                       "/" + self.stompbot.parameters.exchange)
            self.stompbot.send_message(report)

        def on_disconnected(self):
            self.stompbot.logger.debug('Detected disconnect')
            connect_and_subscribe(self.conn, self.stompbot.logger, self.destination)

def connect_and_subscribe(conn, logger, destination):
    conn.start()
    connect_status = conn.connect(wait=True)
    subscribe_status = conn.subscribe(destination=destination,
                                      id=1, ack='auto')
    logger.info('Successfully connected and subscribed. '
                'Connect status: %r, subscribe status: %r.',
                connect_status, subscribe_status)

class StompCollectorBot(CollectorBot):
    """ main class for the STOMP protocol collector """

    collector_empty_process = True

    def init(self):
        if stomp is None:
            raise ValueError('Could not import stomp. Please install it.')

        self.server = getattr(self.parameters, 'server', 'n6stream.cert.pl')
        self.port = getattr(self.parameters, 'port', 61614)
        self.exchange = getattr(self.parameters, 'exchange', '')
        self.heartbeat = getattr(self.parameters, 'heartbeat', 60000)
        self.ssl_ca_cert = getattr(self.parameters, 'ssl_ca_certificate',
                                   'ca.pem')
        self.ssl_cl_cert = getattr(self.parameters, 'ssl_client_certificate',
                                   'client.pem')
        self.ssl_cl_cert_key = getattr(self.parameters,
                                       'ssl_client_certificate_key',
                                       'client.key')
        self.http_verify_cert = getattr(self.parameters,
                                        'http_verify_cert', True)

        # check if certificates exist
        for f in [self.ssl_ca_cert, self.ssl_cl_cert, self.ssl_cl_cert_key]:
            if not os.path.isfile(f):
                raise ValueError("Could not open file %r." % f)

        _host = [(self.server, self.port)]
        self.conn = stomp.Connection(host_and_ports=_host, use_ssl=True,
                                     ssl_key_file=self.ssl_cl_cert_key,
                                     ssl_cert_file=self.ssl_cl_cert,
                                     ssl_ca_certs=self.ssl_ca_cert,
                                     wait_on_receipt=True,
                                     heartbeats=(self.heartbeat,
                                                 self.heartbeat))

        self.conn.set_listener('', StompListener(self, self.conn, self.exchange))
        self.conn.start()
        connect_and_subscribe(self.conn, self.logger, self.exchange)

    def shutdown(self):
        try:
            self.conn.disconnect()
        except stomp.exception.NotConnectedException:
            pass

    def process(self):
        pass

BOT = StompCollectorBot
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""
Generic CSV parser

Parameters:
columns: string
delimiter: string
default_url_protocol: string
skip_header: boolean
type: string
type_translation: string
data_type: string

"""
import csv
import io
import json
import re

from dateutil.parser import parse

from intelmq.lib import utils
from intelmq.lib.bot import ParserBot
from intelmq.lib.exceptions import InvalidArgument, InvalidValue
from intelmq.lib.harmonization import DateTime

TIME_CONVERSIONS = {'timestamp': DateTime.from_timestamp,
                    'windows_nt': DateTime.from_windows_nt,
                    'epoch_millis': DateTime.from_epoch_millis,
                    None: lambda value: parse(value, fuzzy=True).isoformat() + " UTC"}

DATA_CONVERSIONS = {'json': lambda data: json.loads(data)}

class GenericCsvParserBot(ParserBot):

    def init(self):
        self.columns = self.parameters.columns
        # convert columns to an array
        if type(self.columns) is str:
            self.columns = [column.strip() for column in self.columns.split(",")]

        self.type_translation = json.loads(getattr(self.parameters, 'type_translation', None) or '{}')
        self.data_type = json.loads(getattr(self.parameters, 'data_type', None) or '{}')

        # prevents empty strings:
        self.column_regex_search = getattr(self.parameters, 'column_regex_search', None) or {}

        self.time_format = getattr(self.parameters, "time_format", None)
        if self.time_format not in TIME_CONVERSIONS.keys():
            raise InvalidArgument('time_format', got=self.time_format,
                                  expected=list(TIME_CONVERSIONS.keys()),
                                  docs='docs/Bots.md')
        self.filter_text = getattr(self.parameters, 'filter_text', None)
        self.filter_type = getattr(self.parameters, 'filter_type', None)
        if self.filter_type and self.filter_type not in ('blacklist', 'whitelist'):
            raise InvalidArgument('filter_type', got=self.filter_type,
                                  expected=("blacklist", "whitelist"),
                                  docs='docs/Bots.md')
        self.columns_required = getattr(self.parameters, 'columns_required',
                                        [True for _ in self.columns])
        if len(self.columns) != len(self.columns_required):
            raise ValueError("Length of parameters 'columns' (%d) and 'columns_required' (%d) "
                             "needs to be equal." % (len(self.columns), len(self.columns_required)))

    def parse(self, report):
        raw_report = utils.base64_decode(report.get("raw"))
        raw_report = raw_report.translate({0: None})
        # ignore lines starting with #
        raw_report = re.sub(r'(?m)^#.*\n?', '', raw_report)
        # ignore null bytes
        raw_report = re.sub(r'(?m)\0', '', raw_report)
        # skip header
        if getattr(self.parameters, 'skip_header', False):
            raw_report = raw_report[raw_report.find('\n') + 1:]
        for row in csv.reader(io.StringIO(raw_report),
                              delimiter=str(self.parameters.delimiter)):

            if self.filter_text and self.filter_type:
                text_in_row = self.filter_text in self.parameters.delimiter.join(row)
                if text_in_row and self.filter_type == 'whitelist':
                    yield row
                elif not text_in_row and self.filter_type == 'blacklist':
                    yield row
                else:
                    continue
            else:
                    yield row

    def parse_line(self, row, report):
        event = self.new_event(report)

        for keygroup, value, required in zip(self.columns, row, self.columns_required):
            for key in keys:
                if isinstance(value, str) and not value:  # empty string is never valid
                    break
                regex = self.column_regex_search.get(key, None)
                if regex:
                    search = re.search(regex, value)
                    if search:
                        value = search.group(0)
                    else:
                        value = None

                if key in ["__IGNORE__", ""]:
                    break

                if key in self.data_type:
                    value = DATA_CONVERSIONS[self.data_type[key]](value)

                if key in ["time.source", "time.destination"]:
                    value = TIME_CONVERSIONS[self.time_format](value)
                elif key.endswith('.url'):
                    if not value:
                        continue
                    if '://' not in value:
                        value = self.parameters.default_url_protocol + value
                elif key in ["classification.type"] and self.type_translation:
                    if value in self.type_translation:
                        value = self.type_translation[value]
                    elif not hasattr(self.parameters, 'type'):
                        continue
                if event.add(key, value, raise_failure=False):
                    break
            else:
                # if the value sill remains unadded we need to inform if the key is needed
                if required:
                    raise InvalidValue(key, value)

        if hasattr(self.parameters, 'type')\
                and "classification.type" not in event:
            event.add('classification.type', self.parameters.type)
        event.add("raw", self.recover_line(row))
        yield event

    recover_line = ParserBot.recover_line_csv

BOT = GenericCsvParserBot
<commit_msg>BUG: fix merge error breaking generic csv parser<commit_after># -*- coding: utf-8 -*-
"""
Generic CSV parser

Parameters:
columns: string
delimiter: string
default_url_protocol: string
skip_header: boolean
type: string
type_translation: string
data_type: string

"""
import csv
import io
import json
import re

from dateutil.parser import parse

from intelmq.lib import utils
from intelmq.lib.bot import ParserBot
from intelmq.lib.exceptions import InvalidArgument, InvalidValue
from intelmq.lib.harmonization import DateTime

TIME_CONVERSIONS = {'timestamp': DateTime.from_timestamp,
                    'windows_nt': DateTime.from_windows_nt,
                    'epoch_millis': DateTime.from_epoch_millis,
                    None: lambda value: parse(value, fuzzy=True).isoformat() + " UTC"}

DATA_CONVERSIONS = {'json': lambda data: json.loads(data)}

class GenericCsvParserBot(ParserBot):

    def init(self):
        self.columns = self.parameters.columns
        # convert columns to an array
        if type(self.columns) is str:
            self.columns = [column.strip() for column in self.columns.split(",")]

        self.type_translation = json.loads(getattr(self.parameters, 'type_translation', None) or '{}')
        self.data_type = json.loads(getattr(self.parameters, 'data_type', None) or '{}')

        # prevents empty strings:
        self.column_regex_search = getattr(self.parameters, 'column_regex_search', None) or {}

        self.time_format = getattr(self.parameters, "time_format", None)
        if self.time_format not in TIME_CONVERSIONS.keys():
            raise InvalidArgument('time_format', got=self.time_format,
                                  expected=list(TIME_CONVERSIONS.keys()),
                                  docs='docs/Bots.md')
        self.filter_text = getattr(self.parameters, 'filter_text', None)
        self.filter_type = getattr(self.parameters, 'filter_type', None)
        if self.filter_type and self.filter_type not in ('blacklist', 'whitelist'):
            raise InvalidArgument('filter_type', got=self.filter_type,
                                  expected=("blacklist", "whitelist"),
                                  docs='docs/Bots.md')
        self.columns_required = getattr(self.parameters, 'columns_required',
                                        [True for _ in self.columns])
        if len(self.columns) != len(self.columns_required):
            raise ValueError("Length of parameters 'columns' (%d) and 'columns_required' (%d) "
                             "needs to be equal." % (len(self.columns), len(self.columns_required)))

    def parse(self, report):
        raw_report = utils.base64_decode(report.get("raw"))
        raw_report = raw_report.translate({0: None})
        # ignore lines starting with #
        raw_report = re.sub(r'(?m)^#.*\n?', '', raw_report)
        # ignore null bytes
        raw_report = re.sub(r'(?m)\0', '', raw_report)
        # skip header
        if getattr(self.parameters, 'skip_header', False):
            raw_report = raw_report[raw_report.find('\n') + 1:]
        for row in csv.reader(io.StringIO(raw_report),
                              delimiter=str(self.parameters.delimiter)):

            if self.filter_text and self.filter_type:
                text_in_row = self.filter_text in self.parameters.delimiter.join(row)
                if text_in_row and self.filter_type == 'whitelist':
                    yield row
                elif not text_in_row and self.filter_type == 'blacklist':
                    yield row
                else:
                    continue
            else:
                    yield row

    def parse_line(self, row, report):
        event = self.new_event(report)

        for keygroup, value, required in zip(self.columns, row, self.columns_required):
            keys = keygroup.split('|') if '|' in keygroup else [keygroup, ]
            for key in keys:
                if isinstance(value, str) and not value:  # empty string is never valid
                    break
                regex = self.column_regex_search.get(key, None)
                if regex:
                    search = re.search(regex, value)
                    if search:
                        value = search.group(0)
                    else:
                        value = None

                if key in ["__IGNORE__", ""]:
                    break

                if key in self.data_type:
                    value = DATA_CONVERSIONS[self.data_type[key]](value)

                if key in ["time.source", "time.destination"]:
                    value = TIME_CONVERSIONS[self.time_format](value)
                elif key.endswith('.url'):
                    if not value:
                        continue
                    if '://' not in value:
                        value = self.parameters.default_url_protocol + value
                elif key in ["classification.type"] and self.type_translation:
                    if value in self.type_translation:
                        value = self.type_translation[value]
                    elif not hasattr(self.parameters, 'type'):
                        continue
                if event.add(key, value, raise_failure=False):
                    break
            else:
                # if the value sill remains unadded we need to inform if the key is needed
                if required:
                    raise InvalidValue(key, value)

        if hasattr(self.parameters, 'type')\
                and "classification.type" not in event:
            event.add('classification.type', self.parameters.type)
        event.add("raw", self.recover_line(row))
        yield event

    recover_line = ParserBot.recover_line_csv

BOT = GenericCsvParserBot
<|endoftext|>


<commit_before>import collections
import numpy
import six

import chainer
from chainer import cuda
from chainer import function
from chainer import utils
from chainer.utils import type_check
from chainer import variable

def _logsumexp(a, xp, axis=None):
    vmax = xp.amax(a, axis=axis, keepdims=True)
    vmax += xp.log(xp.sum(xp.exp(a - vmax),
                          axis=axis, keepdims=True, dtype=a.dtype))
    return xp.squeeze(vmax, axis=axis)

def _softmax(x, xp):
    val = xp.exp(x - xp.amax(x, axis=2, keepdims=True))
    val /= xp.sum(val, axis=2, keepdims=True)
    return val

def _label_to_path(labels, blank_symbol, xp):
    path = xp.full((len(labels), labels.shape[1] * 2 + 1),
                   blank_symbol, dtype=numpy.int32)
    path[:, 1::2] = labels
    return path

def _log_dot(prob, rr, xp):
    return _logsumexp(prob + xp.swapaxes(rr, 1, 2), xp, axis=2)

def _move_label_to_back(path, path_length, xp):
    s1 = path.shape[1]  # TODO(okuta): Change name
    index = (xp.arange(0, path.size, s1, dtype=numpy.int32)[:, None] +
             (xp.arange(s1) + path_length[:, None])[:, ::-1] % s1)
    return xp.take(path, index)

def _move_inputs(prob, input_length, xp):
    seq, batch, ch = prob.shape
    rotate = (xp.arange(seq)[:, None] + input_length) % seq
    index = rotate * batch + xp.arange(batch)
    return xp.take(prob.reshape(seq * batch, ch), index, axis=0)

class ConnectionistTemporalClassification(function.Function):

    """The implementation of Connectionist Temporal Classfication loss functions.

    To make it usable for real-world cases, this class has two policies below.
    1. This class computes forward and backward variables in the log domain.
    2. This class applies the softmax function to inputs. The Backward
    values of CTC loss is often overflows. This is avoided by computing
    backward values before the activation function is applied.
    """

    def __init__(self, blank_symbol, reduce='mean'):
        self.blank_symbol = blank_symbol
        self.zero_padding = -10000000000.0

        if reduce not in ('mean', 'no'):
            raise ValueError(
                "only 'mean' and 'no' are valid "
                "for 'reduce', but '%s' is given" % reduce)
        self.reduce = reduce

    def check_type_forward(self, in_types):
        type_check.expect(in_types.size() > 3)  # TODO(okuta): > 3?
        l_type = in_types[2]
        type_check.expect(l_type.dtype == numpy.int32)

        x_basetype = in_types[3]  # TODO(oktua): Check x_basetype size

        for i in six.moves.range(3, len(in_types)):
            x_type = in_types[i]
            type_check.expect(
                x_type.dtype == numpy.float32,
                x_type.shape == x_basetype.shape,
            )

    def log_matrix(self, x, xp):
        if xp == numpy:
            res = numpy.ma.log(x).filled(fill_value=self.zero_padding)
        else:
            create_recurrence_relation = cuda.cupy.ElementwiseKernel(
                'T x, T e', 'T y',
                'y = x == 0 ? e : log(x)',
                'create_recurrence_relation')
            res = create_recurrence_relation(x, self.zero_padding)
        return res.astype(numpy.float32)

    def recurrence_relation(self, label, path_length, max_length, dtype, xp):
        """Transition in forword and backword algorithms is represented as matrix.

        See also
        https://blog.wtf.sg/2014/10/06/connectionist-temporal-classification-ctc-with-theano/
        """
        batch, lab = label.shape
        repeat_mask = xp.ones((batch, lab * 2 + 1))
        repeat_mask[:, 1::2] = (label !=
                                xp.take(label, xp.arange(-1, lab - 1)
                                        % lab + xp.arange(0, batch * lab,
                                                          lab)[:, None]))
        repeat_mask[:, 1] = 1
        rr = (xp.eye(max_length, dtype=dtype)[None, :] +
              xp.eye(max_length, k=1, dtype=dtype)[None, :] +
              (xp.eye(max_length, k=2, dtype=dtype) *
               (xp.arange(max_length, dtype=dtype) % dtype(2))[None, :]
               * repeat_mask[:, None]))
        return self.log_matrix(
            rr * (path_length[:, None] > xp.arange(max_length))[..., None], xp)

    # path probablity to label probability
    def label_probability(self, label_size, path, path_length,
                          multiply_seq, xp):
        labels_prob = self.log_matrix(xp.zeros((len(path), label_size),
                                               dtype=multiply_seq.dtype), xp)
        ret = xp.empty(
            (len(multiply_seq),) + labels_prob.shape, dtype=labels_prob.dtype)
        ret[...] = labels_prob
        if xp == numpy:
            for b in six.moves.range(len(path)):
                target_path = path[b][0:path_length[b]]
                chars = {c for c in target_path}
                for c in chars:
                    ret[:, b, c] = _logsumexp(
                        multiply_seq[:, b, 0:path_length[b]]
                        [:, target_path == c], numpy, axis=1)
        else:
            for i, multiply in enumerate(multiply_seq):
                # TODO(okuta): remove loop
                cuda.cupy.ElementwiseKernel(
                    'raw T x, raw I y, raw I l, I b_max, I c_max',
                    'T z',
                    '''
                    T value = z;
                    I c = i % b_max, b = i / b_max;
                    int ind[2] = {b, -1};
                    for (int index = 0; index < c_max; ++index) {
                        ind[1] = index;
                        if (ind[1] < l[ind[0]] && y[ind] == c) {
                            T xvalue = x[ind];
                            T at = xvalue, bt = value;
                            if (value > xvalue) {
                                at = value;
                                bt = xvalue;
                            }
                            value = at + log(1 + exp(bt - at));
                        }
                    }
                    z = value;
                    ''',
                    'reduce_probability')(multiply, path, path_length,
                                          labels_prob.shape[1],
                                          path.shape[1], ret[i])
        return ret

    def calc_trans(self, yseq, input_length,
                   label, label_length, path, path_length, xp):
        forward_prob = self.log_matrix(
            xp.eye(path.shape[1], dtype='f')[0], xp)[None, :]
        backward_prob = forward_prob
        offset = xp.arange(
            0, yseq[0].size, yseq[0].shape[1], dtype=path.dtype)[:, None]

        # prob[i] := forward[i] + backward[-i-1]
        index = offset + path
        frr = self.recurrence_relation(
            label, path_length, path.shape[1], numpy.float32, xp)
        prob = xp.empty(
            (len(yseq),) + index.shape, dtype=forward_prob.dtype)
        # forward computation.
        for i, y in enumerate(yseq):
            # calc forward probability in log scale
            forward_prob = xp.take(y, index) + _log_dot(
                forward_prob[:, None, :], frr, xp)
            prob[i] = forward_prob
        r_index = offset + _move_label_to_back(path, path_length, xp)

        # rotate yseq with path_length
        yseq_inv = _move_inputs(yseq, input_length, xp)[::-1]
        brr = self.recurrence_relation(
            _move_label_to_back(label, label_length, xp),
            path_length, path.shape[1], numpy.float32, xp)
        # move to back.
        prob = _move_inputs(prob, input_length, xp)

        # backward computation.
        ps1 = path.shape[1]
        backward_prob_index = (
            xp.arange(0, path.size, ps1, dtype=numpy.int32)[:, None] +
            (xp.arange(ps1) - path_length[:, None]) % ps1)
        for i, y_inv in enumerate(yseq_inv):
            # calc backward probability
            backward_prob = _log_dot(backward_prob[:, None, :], brr, xp)
            prob[-i - 1] += xp.take(
                backward_prob[:, ::-1], backward_prob_index)
            backward_prob = xp.take(y_inv, r_index) + backward_prob

        # move to front.
        return _move_inputs(prob, -self.input_length, xp)

    def forward(self, inputs):
        xp = cuda.get_array_module(inputs[0])
        self.input_length = inputs[0]
        label_length = inputs[1]
        t = inputs[2]
        xs = inputs[3:]

        if chainer.is_debug():
            # Batch size check.
            assert len(xs[0]) == len(t)
            assert len(xs[0]) == len(self.input_length)
            assert len(xs[0]) == len(label_length)

            # Length check.
            assert len(xs) >= xp.max(self.input_length)
            assert len(t[0]) >= xp.max(label_length)

        self.path_length = 2 * label_length + 1

        yseq_shape = (len(xs),) + xs[0].shape
        self.yseq = _softmax(xp.vstack(xs).reshape(yseq_shape), xp)
        log_yseq = self.log_matrix(self.yseq, xp)
        self.path = _label_to_path(t, self.blank_symbol, xp)
        self.prob_trans = self.calc_trans(
            log_yseq, self.input_length, t,
            label_length, self.path, self.path_length, xp)

        loss = -_logsumexp(self.prob_trans[0], xp, axis=1)
        if self.reduce == 'mean':
            loss = utils.force_array(xp.mean(loss))
        return loss,

    def backward(self, inputs, grad_output):
        xp = cuda.get_array_module(inputs[0])
        batch_size = len(inputs[2])

        total_probability = _logsumexp(self.prob_trans[0], xp, axis=1)
        label_prob = self.label_probability(
            self.yseq.shape[2], self.path, self.path_length,
            self.prob_trans, xp)
        self.yseq -= xp.exp(label_prob - total_probability[:, None])
        if self.reduce == 'mean':
            self.yseq *= grad_output[0] / batch_size
        else:
            self.yseq *= grad_output[0][..., None]
        # mask
        self.yseq *= (
            xp.arange(len(self.yseq))[:, None] < self.input_length)[..., None]
        return (None, None, None) + tuple([y for y in self.yseq])

def connectionist_temporal_classification(
        x, t, blank_symbol, input_length=None, label_length=None,
        reduce='mean'):
    """Connectionist Temporal Classification loss function.

    Connectionist Temporal Classification(CTC) [Graves2006]_ is a loss function
    of sequence labeling where the alignment between the inputs and target is
    unknown. See also [Graves2012]_

    The output is a variable whose value depends on the value of
    the option ``reduce``. If it is ``'no'``, it holds the samplewise
    loss values. If it is ``'mean'``, it takes the mean of loss values.

    Args:
        x (sequence of Variable): RNN output at each time. ``x`` must be a list
            of :class:`~chainer.Variable` s. Each element of ``x``, ``x[i]``
            is a :class:`~chainer.Variable` representing output of RNN at time
            ``i``.
        t (Variable): Expected label sequence.
        blank_symbol (int): Index of blank_symbol.
            This value must be non-negative.
        input_length (Variable): Length of valid sequence for each of mini
            batch ``x`` (optional). If input_length is skipped, It regards that
            all of ``x`` is valid input.
        label_length (Variable): Length of valid sequence for each of mini
            batch ``t`` (optional). If label_length is skipped, It regards that
            all of ``t`` is valid input.
        reduce (str): Reduction option. Its value must be either
            ``'mean'`` or ``'no'``. Otherwise,
            :class:`ValueError` is raised.

    Returns:
       ~chainer.Variable:
           A variable holding a scalar value of the CTC loss.
           If ``reduce`` is ``'no'``, the output variable holds array
           whose shape is `(B,)` where `B` is the number of samples.
           If it is ``'mean'``, it holds a scalar.

    .. note::
       You need to input ``x`` without applying to activation functions(e.g.
       softmax function), because this function applies softmax functions
       to ``x`` before calculating CTC loss to avoid numerical limitations.
       You also need to apply softmax function to forwarded values before you
       decode it.

    .. note::
       This function is differentiable only by ``x``.

    .. note::
       This function supports (batch, sequence, 1-dimensional input)-data.

    .. [Graves2006] Alex Graves, Santiago Fernandez,\
    Faustino Gomez, Jurgen Schmidhuber,\
    `Connectionist Temporal Classification: Labelling Unsegmented\
    Sequence Data with Recurrent Neural Networks\
    <ftp://ftp.idsia.ch/pub/juergen/icml2006.pdf>`_

    .. [Graves2012] Alex Graves,\
    `Supervised Sequence Labelling with Recurrent Neural Networks\
    <http://www.cs.toronto.edu/~graves/preprint.pdf>`_

    """
    if not isinstance(x, collections.Sequence):
        raise TypeError('x must be a list of Variables')
    if not isinstance(blank_symbol, int):
        raise TypeError('blank_symbol must be non-negative integer.')
    assert blank_symbol >= 0
    assert blank_symbol < x[0].shape[1]
    # This implementation only supports 1-dimensional data.
    # TODO(jnishi): Support d(>1)-dimentinal inputs.
    assert(len(x[0].shape) == 2)

    if input_length is None:
        xp = cuda.get_array_module(x[0].data)
        input_length = variable.Variable(
            xp.full((len(x[0].data),), len(x), dtype=numpy.int32))
        label_length = variable.Variable(
            xp.full((len(t.data),), len(t.data[0]), dtype=numpy.int32))

    return ConnectionistTemporalClassification(blank_symbol, reduce)(
        input_length, label_length, t, *x)
<commit_msg>Simplify CTC code<commit_after>import collections
import numpy
import six

import chainer
from chainer import cuda
from chainer import function
from chainer import utils
from chainer.utils import type_check
from chainer import variable

def _logsumexp(a, xp, axis=None):
    vmax = xp.amax(a, axis=axis, keepdims=True)
    vmax += xp.log(xp.sum(xp.exp(a - vmax),
                          axis=axis, keepdims=True, dtype=a.dtype))
    return xp.squeeze(vmax, axis=axis)

def _softmax(x, xp):
    val = xp.exp(x - xp.amax(x, axis=2, keepdims=True))
    val /= xp.sum(val, axis=2, keepdims=True)
    return val

def _label_to_path(labels, blank_symbol, xp):
    path = xp.full((len(labels), labels.shape[1] * 2 + 1),
                   blank_symbol, dtype=numpy.int32)
    path[:, 1::2] = labels
    return path

def _log_dot(prob, rr, xp):
    return _logsumexp(prob + xp.swapaxes(rr, 1, 2), xp, axis=2)

def _move_label_to_back(path, path_length, xp):
    s1 = path.shape[1]  # TODO(okuta): Change name
    index = (xp.arange(0, path.size, s1, dtype=numpy.int32)[:, None] +
             (xp.arange(s1) + path_length[:, None])[:, ::-1] % s1)
    return xp.take(path, index)

def _move_inputs(prob, input_length, xp):
    seq, batch, ch = prob.shape
    rotate = (xp.arange(seq)[:, None] + input_length) % seq
    index = rotate * batch + xp.arange(batch)
    return xp.take(prob.reshape(seq * batch, ch), index, axis=0)

class ConnectionistTemporalClassification(function.Function):

    """The implementation of Connectionist Temporal Classfication loss functions.

    To make it usable for real-world cases, this class has two policies below.
    1. This class computes forward and backward variables in the log domain.
    2. This class applies the softmax function to inputs. The Backward
    values of CTC loss is often overflows. This is avoided by computing
    backward values before the activation function is applied.
    """

    def __init__(self, blank_symbol, reduce='mean'):
        self.blank_symbol = blank_symbol
        self.zero_padding = -10000000000.0

        if reduce not in ('mean', 'no'):
            raise ValueError(
                "only 'mean' and 'no' are valid "
                "for 'reduce', but '%s' is given" % reduce)
        self.reduce = reduce

    def check_type_forward(self, in_types):
        type_check.expect(in_types.size() > 3)  # TODO(okuta): > 3?
        l_type = in_types[2]
        type_check.expect(l_type.dtype == numpy.int32)

        x_basetype = in_types[3]  # TODO(oktua): Check x_basetype size

        for i in six.moves.range(3, len(in_types)):
            x_type = in_types[i]
            type_check.expect(
                x_type.dtype == numpy.float32,
                x_type.shape == x_basetype.shape,
            )

    def log_matrix(self, x, xp):
        if xp == numpy:
            res = numpy.ma.log(x).filled(fill_value=self.zero_padding)
        else:
            create_recurrence_relation = cuda.cupy.ElementwiseKernel(
                'T x, T e', 'T y',
                'y = x == 0 ? e : log(x)',
                'create_recurrence_relation')
            res = create_recurrence_relation(x, self.zero_padding)
        return res.astype(numpy.float32)

    def recurrence_relation(self, label, path_length, max_length, dtype, xp):
        """Transition in forword and backword algorithms is represented as matrix.

        See also
        https://blog.wtf.sg/2014/10/06/connectionist-temporal-classification-ctc-with-theano/
        """
        batch, lab = label.shape
        repeat_mask = xp.ones((batch, lab * 2 + 1))
        repeat_mask[:, 1::2] = (label !=
                                xp.take(label, xp.arange(-1, lab - 1)
                                        % lab + xp.arange(0, batch * lab,
                                                          lab)[:, None]))
        repeat_mask[:, 1] = 1
        rr = (xp.eye(max_length, dtype=dtype)[None, :] +
              xp.eye(max_length, k=1, dtype=dtype)[None, :] +
              (xp.eye(max_length, k=2, dtype=dtype) *
               (xp.arange(max_length, dtype=dtype) % dtype(2))[None, :]
               * repeat_mask[:, None]))
        return self.log_matrix(
            rr * (path_length[:, None] > xp.arange(max_length))[..., None], xp)

    # path probablity to label probability
    def label_probability(self, label_size, path, path_length,
                          multiply_seq, xp):
        labels_prob = self.log_matrix(xp.zeros((len(path), label_size),
                                               dtype=multiply_seq.dtype), xp)
        ret = xp.empty(
            (len(multiply_seq),) + labels_prob.shape, dtype=labels_prob.dtype)
        ret[...] = labels_prob
        if xp == numpy:
            for b in six.moves.range(len(path)):
                target_path = path[b][0:path_length[b]]
                chars = {c for c in target_path}
                for c in chars:
                    ret[:, b, c] = _logsumexp(
                        multiply_seq[:, b, 0:path_length[b]]
                        [:, target_path == c], numpy, axis=1)
        else:
            for i, multiply in enumerate(multiply_seq):
                # TODO(okuta): remove loop
                cuda.cupy.ElementwiseKernel(
                    'raw T x, raw I y, raw I l, I b_max, I c_max',
                    'T z',
                    '''
                    T value = z;
                    I c = i % b_max, b = i / b_max;
                    int ind[2] = {b, -1};
                    for (int index = 0; index < c_max; ++index) {
                        ind[1] = index;
                        if (ind[1] < l[ind[0]] && y[ind] == c) {
                            T xvalue = x[ind];
                            T at = xvalue, bt = value;
                            if (value > xvalue) {
                                at = value;
                                bt = xvalue;
                            }
                            value = at + log(1 + exp(bt - at));
                        }
                    }
                    z = value;
                    ''',
                    'reduce_probability')(multiply, path, path_length,
                                          labels_prob.shape[1],
                                          path.shape[1], ret[i])
        return ret

    def calc_trans(self, yseq, input_length,
                   label, label_length, path, path_length, xp):
        forward_prob = self.log_matrix(
            xp.eye(path.shape[1], dtype='f')[0], xp)[None, :]
        backward_prob = forward_prob
        offset = xp.arange(
            0, yseq[0].size, yseq[0].shape[1], dtype=path.dtype)[:, None]

        # prob[i] := forward[i] + backward[-i-1]
        index = offset + path
        frr = self.recurrence_relation(
            label, path_length, path.shape[1], numpy.float32, xp)
        prob = xp.empty(
            (len(yseq),) + index.shape, dtype=forward_prob.dtype)
        # forward computation.
        for i, y in enumerate(yseq):
            # calc forward probability in log scale
            forward_prob = xp.take(y, index) + _log_dot(
                forward_prob[:, None, :], frr, xp)
            prob[i] = forward_prob
        r_index = offset + _move_label_to_back(path, path_length, xp)

        # rotate yseq with path_length
        yseq_inv = _move_inputs(yseq, input_length, xp)[::-1]
        brr = self.recurrence_relation(
            _move_label_to_back(label, label_length, xp),
            path_length, path.shape[1], numpy.float32, xp)
        # move to back.
        prob = _move_inputs(prob, input_length, xp)

        # backward computation.
        ps1 = path.shape[1]
        backward_prob_index = (
            xp.arange(0, path.size, ps1, dtype=numpy.int32)[:, None] +
            (xp.arange(ps1) - path_length[:, None]) % ps1)
        for i, y_inv in enumerate(yseq_inv):
            # calc backward probability
            backward_prob = _log_dot(backward_prob[:, None, :], brr, xp)
            prob[-i - 1] += xp.take(
                backward_prob[:, ::-1], backward_prob_index)
            backward_prob = xp.take(y_inv, r_index) + backward_prob

        # move to front.
        return _move_inputs(prob, -self.input_length, xp)

    def forward(self, inputs):
        xp = cuda.get_array_module(inputs[0])
        self.input_length = inputs[0]
        label_length = inputs[1]
        t = inputs[2]
        xs = inputs[3:]

        if chainer.is_debug():
            # Batch size check.
            assert len(xs[0]) == len(t)
            assert len(xs[0]) == len(self.input_length)
            assert len(xs[0]) == len(label_length)

            # Length check.
            assert len(xs) >= xp.max(self.input_length)
            assert len(t[0]) >= xp.max(label_length)

        self.path_length = 2 * label_length + 1

        yseq_shape = (len(xs),) + xs[0].shape
        self.yseq = _softmax(xp.vstack(xs).reshape(yseq_shape), xp)
        log_yseq = self.log_matrix(self.yseq, xp)
        self.path = _label_to_path(t, self.blank_symbol, xp)
        self.prob_trans = self.calc_trans(
            log_yseq, self.input_length, t,
            label_length, self.path, self.path_length, xp)

        loss = -_logsumexp(self.prob_trans[0], xp, axis=1)
        if self.reduce == 'mean':
            loss = utils.force_array(xp.mean(loss))
        return loss,

    def backward(self, inputs, grad_output):
        xp = cuda.get_array_module(inputs[0])
        batch_size = len(inputs[2])

        total_probability = _logsumexp(self.prob_trans[0], xp, axis=1)
        label_prob = self.label_probability(
            self.yseq.shape[2], self.path, self.path_length,
            self.prob_trans, xp)
        self.yseq -= xp.exp(label_prob - total_probability[:, None])
        if self.reduce == 'mean':
            self.yseq *= grad_output[0] / batch_size
        else:
            self.yseq *= grad_output[0][..., None]
        # mask
        self.yseq *= (
            xp.arange(len(self.yseq))[:, None] < self.input_length)[..., None]
        return (None, None, None) + tuple([y for y in self.yseq])

def connectionist_temporal_classification(
        x, t, blank_symbol, input_length=None, label_length=None,
        reduce='mean'):
    """Connectionist Temporal Classification loss function.

    Connectionist Temporal Classification(CTC) [Graves2006]_ is a loss function
    of sequence labeling where the alignment between the inputs and target is
    unknown. See also [Graves2012]_

    The output is a variable whose value depends on the value of
    the option ``reduce``. If it is ``'no'``, it holds the samplewise
    loss values. If it is ``'mean'``, it takes the mean of loss values.

    Args:
        x (sequence of Variable): RNN output at each time. ``x`` must be a list
            of :class:`~chainer.Variable` s. Each element of ``x``, ``x[i]``
            is a :class:`~chainer.Variable` representing output of RNN at time
            ``i``.
        t (Variable): Expected label sequence.
        blank_symbol (int): Index of blank_symbol.
            This value must be non-negative.
        input_length (Variable): Length of valid sequence for each of mini
            batch ``x`` (optional). If input_length is skipped, It regards that
            all of ``x`` is valid input.
        label_length (Variable): Length of valid sequence for each of mini
            batch ``t`` (optional). If label_length is skipped, It regards that
            all of ``t`` is valid input.
        reduce (str): Reduction option. Its value must be either
            ``'mean'`` or ``'no'``. Otherwise,
            :class:`ValueError` is raised.

    Returns:
       ~chainer.Variable:
           A variable holding a scalar value of the CTC loss.
           If ``reduce`` is ``'no'``, the output variable holds array
           whose shape is `(B,)` where `B` is the number of samples.
           If it is ``'mean'``, it holds a scalar.

    .. note::
       You need to input ``x`` without applying to activation functions(e.g.
       softmax function), because this function applies softmax functions
       to ``x`` before calculating CTC loss to avoid numerical limitations.
       You also need to apply softmax function to forwarded values before you
       decode it.

    .. note::
       This function is differentiable only by ``x``.

    .. note::
       This function supports (batch, sequence, 1-dimensional input)-data.

    .. [Graves2006] Alex Graves, Santiago Fernandez,\
    Faustino Gomez, Jurgen Schmidhuber,\
    `Connectionist Temporal Classification: Labelling Unsegmented\
    Sequence Data with Recurrent Neural Networks\
    <ftp://ftp.idsia.ch/pub/juergen/icml2006.pdf>`_

    .. [Graves2012] Alex Graves,\
    `Supervised Sequence Labelling with Recurrent Neural Networks\
    <http://www.cs.toronto.edu/~graves/preprint.pdf>`_

    """
    if not isinstance(x, collections.Sequence):
        raise TypeError('x must be a list of Variables')
    if not isinstance(blank_symbol, int):
        raise TypeError('blank_symbol must be non-negative integer.')
    assert 0 <= blank_symbol < x[0].shape[1]
    # This implementation only supports 1-dimensional data.
    # TODO(jnishi): Support d(>1)-dimentinal inputs.
    assert(x[0].ndim == 2)

    if input_length is None:
        xp = cuda.get_array_module(x[0])
        input_length = xp.full(len(x[0]), len(x), dtype=numpy.int32)
        label_length = xp.full(len(t), t.shape[1], dtype=numpy.int32)

    return ConnectionistTemporalClassification(blank_symbol, reduce)(
        input_length, label_length, t, *x)
<|endoftext|>


<commit_before>#!/usr/bin/env python

from time import sleep
import socket
import sys
import os
import Queue
import logging
logging.basicConfig(level=logging.INFO, format="%(message)s")
logger = logging.getLogger('program')

logger.info("-" * 80)
logger.info("-- M3 Programmer")
logger.info("")

from ice import ICE
ice = ICE()

if len(sys.argv) not in (2,):
    logger.info("USAGE: %s SERIAL_DEVICE\n" % (sys.argv[0]))
    sys.exit(2)

# Callback for async I2C message
def async_helper(msg_type, event_id, length, msg):
    logger.debug("Bin Helper got msg len" + str(len(msg)))
    if len(msg) == 0:
        logger.debug("Ignore msg of len 0")
        return
    #msq_q.put(msg)
    logger.info("Got msg: " + msg.encode('hex'))

#msq_q = Queue.Queue()
ice.msg_handler['d+'] = async_helper

ice.connect(sys.argv[1])
ice.i2c_set_address("1001100x") # 0x98

resp = raw_input("Would you like to send the DMA start interrupt? [Y/n] ")
if len(resp) != 0 and resp[0] in ('n', 'N'):
    sys.exit()

logger.info("Sending 0x88 0x00000000")
ice.i2c_send(0x88, "00000000".decode('hex'))

logger.info("Listening...")
resp = raw_input("Press enter at any time to quit\n")
<commit_msg>Remove old file<commit_after><|endoftext|>


<commit_before>""" Flask routing """
# pylint: disable=no-member
from chartingperformance import app
from chartingperformance import db_session

from chartingperformance.models import Houses
from chartingperformance.models import MonitorDevices
from chartingperformance.models import EnergyMonthly
from chartingperformance.models import EnergyHourly
from chartingperformance.models import LimitsHourly

from chartingperformance.views import CircuitDict
from chartingperformance.views import ViewSummary
from chartingperformance.views import ViewGeneration
from chartingperformance.views import ViewUsage
from chartingperformance.views import ViewHdd
from chartingperformance.views import ViewWater
from chartingperformance.views import ViewBasetemp
from chartingperformance.views import ViewHeatmap
from chartingperformance.views import ViewChart

from flask import request, jsonify, url_for, make_response

from sqlalchemy import func

@app.errorhandler(404)
def not_found(error):
    """ Return page not found error in json. """
    return make_response(jsonify({'error': 'Not found'}), 404)

@app.route('/api/houses/', methods=['GET'])
def get_houses():
    """ Return list of houses. """

    return jsonify(houses=get_houses_all())

@app.route('/api/houses/<house_id>/', methods=['GET'])
def get_house(house_id):
    """ Return details for house X. """

    return jsonify(house=get_house_details(house_id))

@app.route('/api/houses/<house_id>/devices/', methods=['GET'])
def get_devices(house_id):
    """ Return list of devices for house X. """

    return jsonify(devices=get_devices_all(house_id))

@app.route('/api/houses/<house_id>/circuits/', methods=['GET'])
def get_circuits(house_id):
    """ Return list of circuits for house X. """

    circuits = CircuitDict(house_id)

    return circuits.get_response()

@app.route('/api/houses/<house_id>/views/default/', methods=['GET'])
def setup(house_id):
    """ Return default values for houses X views. """

    interval = request.args.get('interval', 'months')

    if 'month' in interval:
        return jsonify(years=get_years(house_id),
                       asof=get_asof_date(house_id),
                       house=get_house_details(house_id))
    elif 'day' in interval:
        return jsonify(limits=get_limits(house_id))
    else:
        return jsonify(error='Interval \'%s\' does not exist' % interval)

@app.route('/api/houses/<house_id>/views/summary/', methods=['GET'])
def view_summary(house_id):
    """ Return summary view for house X. """

    view = ViewSummary(request.args, house_id)

    return view.get_response()

@app.route('/api/houses/<house_id>/views/generation/', methods=['GET'])
def view_generation(house_id):
    """ Return generation view for house X. """

    view = ViewGeneration(request.args, house_id)

    return view.get_response()

@app.route('/api/houses/<house_id>/views/usage/', methods=['GET'])
def view_usage(house_id):
    """ Return usage view for house X. """

    view = ViewUsage(request.args, house_id)

    return view.get_response()

@app.route('/api/houses/<house_id>/views/hdd/', methods=['GET'])
def view_hdd(house_id):
    """ Return hdd view for house X. """

    view = ViewHdd(request.args, house_id)

    return view.get_response()

@app.route('/api/houses/<house_id>/views/water/', methods=['GET'])
def view_water(house_id):
    """ Return water view for house X. """

    view = ViewWater(request.args, house_id)

    return view.get_response()

@app.route('/api/houses/<house_id>/views/basetemp/', methods=['GET'])
def view_heat(house_id):
    """ Return basetemp view for house X. """

    view = ViewBasetemp(request.args, house_id)

    return view.get_response()

@app.route('/api/houses/<house_id>/views/heatmap/', methods=['GET'])
def view_heatmap(house_id):
    """ Return heatmap daily view for house X. """

    view = ViewHeatmap(request.args, house_id)

    return view.get_response()

@app.route('/api/houses/<house_id>/views/chart/', methods=['GET'])
def view_chart(house_id):
    """ Return chart hourly view for house X. """

    view = ViewChart(request.args, house_id)

    return view.get_response()

def get_houses_all():
    """ Return array of houses. """

    houses = db_session.query(Houses).all()

    json_items = []
    for house in houses:
        data = {'name': house.name,
                'sname': house.sname,
                'house_id': house.house_id,
                'url': url_for('get_house', house_id=house.house_id, _external=True)}
        json_items.append(data)

    return json_items

def get_house_details(house_id):
    """ Return details for house X in json. """

    house = db_session.query(Houses). \
        filter(Houses.house_id == house_id).one()

    return {'name': house.name,
            'sname': house.sname,
            'id': house.house_id,
            'devices': url_for('get_devices', house_id=house.house_id, _external=True),
            'circuits': url_for('get_circuits', house_id=house.house_id, _external=True)}

def get_devices_all(house_id):
    """ Return array of devices for house X. """

    devices = db_session.query(MonitorDevices). \
        filter(Houses.house_id == house_id).all()

    json_items = []
    for device in devices:
        data = {'name': device.name,
                'id': device.device_id}
        json_items.append(data)

    return json_items

def get_years(house_id):
    """ Return array of valid years of energy data for house X. """

    years = db_session.query(EnergyMonthly.date). \
        filter(EnergyMonthly.house_id == house_id). \
        group_by(func.year(EnergyMonthly.date)). \
        order_by(EnergyMonthly.date)

    json_items = []
    for year in years:
        json_items.append(str(year[0].year))

    return json_items

def get_asof_date(house_id):
    """ Return latest date of data as string for house X. """

    asof = db_session.query(LimitsHourly.end_date). \
        filter(LimitsHourly.house_id == house_id).one()

    return str(asof.end_date.strftime("%Y-%m-%d"))

def get_limits(house_id):
    """ Return deails of data limits for house X. Used mainlt in chart view. """

    limits = db_session.query(LimitsHourly). \
        filter(EnergyHourly.house_id == house_id).one()

    return {'used_max': limits.used_max,
            'solar_min': limits.solar_min,
            'outdoor_deg_min': limits.outdoor_deg_min,
            'outdoor_deg_max': limits.outdoor_deg_max,
            'hdd_max': limits.hdd_max,
            'start_date': str(limits.start_date),
            'end_date': str(limits.end_date)}
<commit_msg>Added /views endpoint<commit_after>""" Flask routing """
# pylint: disable=no-member
from chartingperformance import app
from chartingperformance import db_session

from chartingperformance.models import Houses
from chartingperformance.models import MonitorDevices
from chartingperformance.models import EnergyMonthly
from chartingperformance.models import EnergyHourly
from chartingperformance.models import LimitsHourly

from chartingperformance.views import CircuitDict
from chartingperformance.views import ViewSummary
from chartingperformance.views import ViewGeneration
from chartingperformance.views import ViewUsage
from chartingperformance.views import ViewHdd
from chartingperformance.views import ViewWater
from chartingperformance.views import ViewBasetemp
from chartingperformance.views import ViewHeatmap
from chartingperformance.views import ViewChart

from flask import request, jsonify, url_for, make_response

from sqlalchemy import func

@app.errorhandler(404)
def not_found(error):
    """ Return page not found error in json. """
    return make_response(jsonify({'error': 'Not found'}), 404)

@app.route('/api/houses/', methods=['GET'])
def get_houses():
    """ Return list of houses. """

    return jsonify(houses=get_houses_all())

@app.route('/api/houses/<house_id>/', methods=['GET'])
def get_house(house_id):
    """ Return details for house X. """

    return jsonify(house=get_house_details(house_id))

@app.route('/api/houses/<house_id>/devices/', methods=['GET'])
def get_devices(house_id):
    """ Return list of devices for house X. """

    return jsonify(devices=get_devices_all(house_id))

@app.route('/api/houses/<house_id>/circuits/', methods=['GET'])
def get_circuits(house_id):
    """ Return list of circuits for house X. """

    circuits = CircuitDict(house_id)

    return circuits.get_response()

@app.route('/api/houses/<house_id>/views/', methods=['GET'])
def views(house_id):
    """ Return list of available views for house X. """

    return jsonify(views=get_views(house_id))

@app.route('/api/houses/<house_id>/views/default/', methods=['GET'])
def setup(house_id):
    """ Return default values for houses X views. """

    interval = request.args.get('interval', 'months')

    if 'month' in interval:
        return jsonify(years=get_years(house_id),
                       asof=get_asof_date(house_id),
                       house=get_house_details(house_id))
    elif 'day' in interval:
        return jsonify(limits=get_limits(house_id))
    else:
        return jsonify(error='Interval \'%s\' does not exist' % interval)

@app.route('/api/houses/<house_id>/views/summary/', methods=['GET'])
def view_summary(house_id):
    """ Return summary view for house X. """

    view = ViewSummary(request.args, house_id)

    return view.get_response()

@app.route('/api/houses/<house_id>/views/generation/', methods=['GET'])
def view_generation(house_id):
    """ Return generation view for house X. """

    view = ViewGeneration(request.args, house_id)

    return view.get_response()

@app.route('/api/houses/<house_id>/views/usage/', methods=['GET'])
def view_usage(house_id):
    """ Return usage view for house X. """

    view = ViewUsage(request.args, house_id)

    return view.get_response()

@app.route('/api/houses/<house_id>/views/hdd/', methods=['GET'])
def view_hdd(house_id):
    """ Return hdd view for house X. """

    view = ViewHdd(request.args, house_id)

    return view.get_response()

@app.route('/api/houses/<house_id>/views/water/', methods=['GET'])
def view_water(house_id):
    """ Return water view for house X. """

    view = ViewWater(request.args, house_id)

    return view.get_response()

@app.route('/api/houses/<house_id>/views/basetemp/', methods=['GET'])
def view_heat(house_id):
    """ Return basetemp view for house X. """

    view = ViewBasetemp(request.args, house_id)

    return view.get_response()

@app.route('/api/houses/<house_id>/views/heatmap/', methods=['GET'])
def view_heatmap(house_id):
    """ Return heatmap daily view for house X. """

    view = ViewHeatmap(request.args, house_id)

    return view.get_response()

@app.route('/api/houses/<house_id>/views/chart/', methods=['GET'])
def view_chart(house_id):
    """ Return chart hourly view for house X. """

    view = ViewChart(request.args, house_id)

    return view.get_response()

def get_houses_all():
    """ Return array of houses. """

    houses = db_session.query(Houses).all()

    json_items = []
    for house in houses:
        data = {'name': house.name,
                'sname': house.sname,
                'house_id': house.house_id,
                'url': url_for('get_house', house_id=house.house_id, _external=True)}
        json_items.append(data)

    return json_items

def get_house_details(house_id):
    """ Return details for house X in json. """

    house = db_session.query(Houses). \
        filter(Houses.house_id == house_id).one()

    return {'name': house.name,
            'sname': house.sname,
            'id': house.house_id,
            'devices': url_for('get_devices', house_id=house.house_id, _external=True),
            'circuits': url_for('get_circuits', house_id=house.house_id, _external=True),
            'views': url_for('views', house_id=house.house_id, _external=True)}

def get_devices_all(house_id):
    """ Return array of devices for house X. """

    devices = db_session.query(MonitorDevices). \
        filter(Houses.house_id == house_id).all()

    json_items = []
    for device in devices:
        data = {'name': device.name,
                'id': device.device_id}
        json_items.append(data)

    return json_items

def get_years(house_id):
    """ Return array of valid years of energy data for house X. """

    years = db_session.query(EnergyMonthly.date). \
        filter(EnergyMonthly.house_id == house_id). \
        group_by(func.year(EnergyMonthly.date)). \
        order_by(EnergyMonthly.date)

    json_items = []
    for year in years:
        json_items.append(str(year[0].year))

    return json_items

def get_asof_date(house_id):
    """ Return latest date of data as string for house X. """

    asof = db_session.query(LimitsHourly.end_date). \
        filter(LimitsHourly.house_id == house_id).one()

    return str(asof.end_date.strftime("%Y-%m-%d"))

def get_limits(house_id):
    """ Return deails of data limits for house X. Used mainlt in chart view. """

    limits = db_session.query(LimitsHourly). \
        filter(EnergyHourly.house_id == house_id).one()

    return {'used_max': limits.used_max,
            'solar_min': limits.solar_min,
            'outdoor_deg_min': limits.outdoor_deg_min,
            'outdoor_deg_max': limits.outdoor_deg_max,
            'hdd_max': limits.hdd_max,
            'start_date': str(limits.start_date),
            'end_date': str(limits.end_date)}

def get_views(house_id):
    """  Return list of endpoints """

    return [
        url_for('setup', house_id=house_id, _external=True),
        url_for('view_summary', house_id=house_id, _external=True),
        url_for('view_generation', house_id=house_id, _external=True),
        url_for('view_usage', house_id=house_id, _external=True),
        url_for('view_hdd', house_id=house_id, _external=True),
        url_for('view_water', house_id=house_id, _external=True),
        url_for('view_heatmap', house_id=house_id, _external=True),
        url_for('view_chart', house_id=house_id, _external=True)
    ]<|endoftext|>


<commit_before>import aiohttp
from sanic.log import log

HOST = '127.0.0.1'
PORT = 42101

async def local_request(method, uri, cookies=None, *args, **kwargs):
    url = 'http://{host}:{port}{uri}'.format(host=HOST, port=PORT, uri=uri)
    log.info(url)
    async with aiohttp.ClientSession(cookies=cookies) as session:
        async with getattr(session, method)(url, *args, **kwargs) as response:
            response.text = await response.text()
            response.body = await response.read()
            return response

def sanic_endpoint_test(app, method='get', uri='/', gather_request=True,
                        loop=None, debug=False, server_kwargs={},
                        *request_args, **request_kwargs):
    results = []
    exceptions = []

    if gather_request:
        @app.middleware
        def _collect_request(request):
            results.append(request)

    async def _collect_response(sanic, loop):
        try:
            response = await local_request(method, uri, *request_args,
                                           **request_kwargs)
            results.append(response)
        except Exception as e:
            exceptions.append(e)
        app.stop()

    app.run(host=HOST, debug=debug, port=42101,
            after_start=_collect_response, loop=loop, **server_kwargs)

    if exceptions:
        raise ValueError("Exception during request: {}".format(exceptions))

    if gather_request:
        try:
            request, response = results
            return request, response
        except:
            raise ValueError(
                "Request and response object expected, got ({})".format(
                    results))
    else:
        try:
            return results[0]
        except:
            raise ValueError(
                "Request object expected, got ({})".format(results))
<commit_msg>Use constant PORT rather than literal in test code (#266)<commit_after>import aiohttp
from sanic.log import log

HOST = '127.0.0.1'
PORT = 42101

async def local_request(method, uri, cookies=None, *args, **kwargs):
    url = 'http://{host}:{port}{uri}'.format(host=HOST, port=PORT, uri=uri)
    log.info(url)
    async with aiohttp.ClientSession(cookies=cookies) as session:
        async with getattr(session, method)(url, *args, **kwargs) as response:
            response.text = await response.text()
            response.body = await response.read()
            return response

def sanic_endpoint_test(app, method='get', uri='/', gather_request=True,
                        loop=None, debug=False, server_kwargs={},
                        *request_args, **request_kwargs):
    results = []
    exceptions = []

    if gather_request:
        @app.middleware
        def _collect_request(request):
            results.append(request)

    async def _collect_response(sanic, loop):
        try:
            response = await local_request(method, uri, *request_args,
                                           **request_kwargs)
            results.append(response)
        except Exception as e:
            exceptions.append(e)
        app.stop()

    app.run(host=HOST, debug=debug, port=PORT,
            after_start=_collect_response, loop=loop, **server_kwargs)

    if exceptions:
        raise ValueError("Exception during request: {}".format(exceptions))

    if gather_request:
        try:
            request, response = results
            return request, response
        except:
            raise ValueError(
                "Request and response object expected, got ({})".format(
                    results))
    else:
        try:
            return results[0]
        except:
            raise ValueError(
                "Request object expected, got ({})".format(results))
<|endoftext|>


<commit_before>#!/usr/bin/env python3
#
# Copyright 2016 Red Hat, Inc.
#
# Authors:
#     Fam Zheng <famz@redhat.com>
#
# This work is licensed under the MIT License.  Please see the LICENSE file or
# http://opensource.org/licenses/MIT.

from .models import Message, Result
from django.db.models import Q

class InvalidSearchTerm(Exception):
    pass

class SearchEngine(object):
    """

The general form of search string is a list of terms separated with space:

    QUERY = TERM TERM ...

Each term can be either a plain keyword, or a predict in the form of
`PRED:EXP`, where PRED is the predefined filter and EXP is the parameters to be
applied to the filter. As a simple example:

    bugfix from:Bob to:George age:>1w

to search emails titled as 'bugfix' (a subject keyword filter) from Bob (a
sender filter) to George (a recipient filter) before 1 week ago (an age
filter).

or

    bugfix from:Bob is:reviewed not:obsoleted

to search all emails from Bob that have "bugfix" in subject, and have been
reviewed but is not obsoleted (by a new revision of this series). Because there
are syntax shortcut for some predicts, it can be simplified as:

    from:Bob fix +reviewed -tested

---

## Supported filter types

### Search by age

 - Syntax: age:AGE
 - Syntax: >AGE
 - Syntax: <AGE

Filter by age of the message. Supports "d" (day), "w" (week), "m" (month) and "y" (year) as units. Examples:

 - age:1d
 - age:>2d
 - age:<1w
 - <1m
 - \>1w

---

### Search by series state

Syntax:

 - is:reviewed - all the patches in the series is reviewed
 - is:obsolete or is:old - the series has newer version
 - is:complete - the series has all the patches it contains
 - is:merged - the series is included in the project's git tree
 - is:pull - the series is a pull request
 - is:applied - a git tree is available for the series
 - has:replies - the series received a reply (apart from patches sent by the submitter)

Example:

    is:reviewed

---

### Search addresses

 - Syntax: from:ADDRESS
 - Syntax: to:ADDRESS

Compare the address info of message. Example:

    from:alice to:bob

---

### Reverse condition

 - Syntax: !TERM

Negative of an expression. Example:

    !is:reviewed     (query series that are not reviewed)
    !has:replies     (query series that have not received any comment)

---

### Search by message id

 - Syntax: id:MESSAGE-ID

Exact match of message-id. Example:

    id:<1416902879-17422-1-git-send-email-user@domain.com>

or

    id:1416902879-17422-1-git-send-email-user@domain.com

---

### Search by text

 - Syntax: KEYWORD

Search text keyword in the email message. Example:

    regression

"""
    def _process_term(self, query, term, neg=False):
        """ Return a Q object that will be applied to the query """
        def as_keywords(t):
            self._last_keywords.append(t)
            return Q(subject__icontains=t)

        if term.startswith("!"):
            return self._process_term(query, term[1:], not neg)
        if term.startswith("age:"):
            cond = term[term.find(":") + 1:]
            q = self._process_age_term(query, cond)
        elif term[0] in "<>" and len(term) > 1:
            q = self._process_age_term(query, term)
        elif term.startswith("from:"):
            cond = term[term.find(":") + 1:]
            q = Q(sender__icontains=cond)
        elif term.startswith("to:"):
            cond = term[term.find(":") + 1:]
            q = Q(recipients__icontains=cond)
        elif term.startswith("subject:"):
            cond = term[term.find(":") + 1:]
            q = Q(subject__icontains=cond)
        elif term.startswith("id:"):
            cond = term[term.find(":") + 1:]
            if cond[0] == "<" and cond[-1] == ">":
                cond = cond[1:-1]
            q = Q(message_id=cond)
        elif term.startswith("is:") or term.startswith("not:") or term[0] in "+-":
            if term[0] in "+-":
                cond = term[1:]
                lneg = term[0] == "-"
            else:
                cond = term[term.find(":") + 1:]
                lneg = term.startswith("not:")
            if cond == "complete":
                q = Q(is_complete=True)
            elif cond == "pull":
                q = Q(subject__contains='[PULL') | Q(subject__contains='[GIT PULL')
            elif cond == "reviewed":
                q = Q(properties__name="reviewed",
                      properties__value="true")
            elif cond in ("obsoleted", "old"):
                q = Q(properties__name="obsoleted-by",
                      properties__value__isnull=False) & \
                    ~Q(properties__name="obsoleted-by",
                      properties__value__iexact='')
            elif cond == "applied":
                q = Q(results__name="git",
                      results__status=Result.SUCCESS)
            elif cond == "tested":
                q = Q(properties__name="testing.done",
                      properties__value="true")
            elif cond == "merged":
                q = Q(is_merged=True)
            else:
                q = as_keywords(term)
            if lneg:
                neg = not neg
        elif term.startswith("has:"):
            cond = term[term.find(":") + 1:]
            if cond == "replies":
                q = Q(last_comment_date__isnull=False)
            else:
                q = Q(properties__name=cond)
        elif term.startswith("project:"):
            cond = term[term.find(":") + 1:]
            self._projects.add(cond)
            q = Q(project__name=cond) | Q(project__parent_project__name=cond)
        else:
            # Keyword in subject is the default
            q = as_keywords(term)
        if neg:
            return query.exclude(pk__in=query.filter(q))
        else:
            return query.filter(q)

    def last_keywords(self):
        return getattr(self, "_last_keywords", [])

    def project(self):
        return next(iter(self._projects)) if len(self._projects) == 1 else None

    def search_series(self, *terms, queryset=None):
        self._last_keywords = []
        self._projects = set()
        if queryset is None:
            queryset = Message.objects.series_heads()
        for t in terms:
            queryset = self._process_term(queryset, t)
        return queryset

    def _process_age_term(self, query, cond):
        import datetime
        def human_to_seconds(n, unit):
            if unit == "d":
                return n * 86400
            elif unit == "w":
                return n * 86400 * 7
            elif unit == "m":
                return n * 86400 * 30
            elif unit == "y":
                return n * 86400 * 365
            raise Exception("No unit specified")

        if cond.startswith("<"):
            less = True
            cond = cond[1:]
        elif cond.startswith(">"):
            less = False
            cond = cond[1:]
        else:
            less = False
        num, unit = cond[:-1], cond[-1].lower()
        if not num.isdigit() or not unit in "dwmy":
            raise InvalidSearchTerm("Invalid age string: %s" % cond)
        sec = human_to_seconds(int(num), unit)
        p = datetime.datetime.now() - datetime.timedelta(0, sec)
        if less:
            q = Q(date__gte=p)
        else:
            q = Q(date__lte=p)
        return q
<commit_msg>search: fix matches on property value<commit_after>#!/usr/bin/env python3
#
# Copyright 2016 Red Hat, Inc.
#
# Authors:
#     Fam Zheng <famz@redhat.com>
#
# This work is licensed under the MIT License.  Please see the LICENSE file or
# http://opensource.org/licenses/MIT.

from .models import Message, Result
from django.db.models import Q

class InvalidSearchTerm(Exception):
    pass

class SearchEngine(object):
    """

The general form of search string is a list of terms separated with space:

    QUERY = TERM TERM ...

Each term can be either a plain keyword, or a predict in the form of
`PRED:EXP`, where PRED is the predefined filter and EXP is the parameters to be
applied to the filter. As a simple example:

    bugfix from:Bob to:George age:>1w

to search emails titled as 'bugfix' (a subject keyword filter) from Bob (a
sender filter) to George (a recipient filter) before 1 week ago (an age
filter).

or

    bugfix from:Bob is:reviewed not:obsoleted

to search all emails from Bob that have "bugfix" in subject, and have been
reviewed but is not obsoleted (by a new revision of this series). Because there
are syntax shortcut for some predicts, it can be simplified as:

    from:Bob fix +reviewed -tested

---

## Supported filter types

### Search by age

 - Syntax: age:AGE
 - Syntax: >AGE
 - Syntax: <AGE

Filter by age of the message. Supports "d" (day), "w" (week), "m" (month) and "y" (year) as units. Examples:

 - age:1d
 - age:>2d
 - age:<1w
 - <1m
 - \>1w

---

### Search by series state

Syntax:

 - is:reviewed - all the patches in the series is reviewed
 - is:obsolete or is:old - the series has newer version
 - is:complete - the series has all the patches it contains
 - is:merged - the series is included in the project's git tree
 - is:pull - the series is a pull request
 - is:applied - a git tree is available for the series
 - has:replies - the series received a reply (apart from patches sent by the submitter)

Example:

    is:reviewed

---

### Search addresses

 - Syntax: from:ADDRESS
 - Syntax: to:ADDRESS

Compare the address info of message. Example:

    from:alice to:bob

---

### Reverse condition

 - Syntax: !TERM

Negative of an expression. Example:

    !is:reviewed     (query series that are not reviewed)
    !has:replies     (query series that have not received any comment)

---

### Search by message id

 - Syntax: id:MESSAGE-ID

Exact match of message-id. Example:

    id:<1416902879-17422-1-git-send-email-user@domain.com>

or

    id:1416902879-17422-1-git-send-email-user@domain.com

---

### Search by text

 - Syntax: KEYWORD

Search text keyword in the email message. Example:

    regression

"""
    def _process_term(self, query, term, neg=False):
        """ Return a Q object that will be applied to the query """
        def as_keywords(t):
            self._last_keywords.append(t)
            return Q(subject__icontains=t)

        if term.startswith("!"):
            return self._process_term(query, term[1:], not neg)
        if term.startswith("age:"):
            cond = term[term.find(":") + 1:]
            q = self._process_age_term(query, cond)
        elif term[0] in "<>" and len(term) > 1:
            q = self._process_age_term(query, term)
        elif term.startswith("from:"):
            cond = term[term.find(":") + 1:]
            q = Q(sender__icontains=cond)
        elif term.startswith("to:"):
            cond = term[term.find(":") + 1:]
            q = Q(recipients__icontains=cond)
        elif term.startswith("subject:"):
            cond = term[term.find(":") + 1:]
            q = Q(subject__icontains=cond)
        elif term.startswith("id:"):
            cond = term[term.find(":") + 1:]
            if cond[0] == "<" and cond[-1] == ">":
                cond = cond[1:-1]
            q = Q(message_id=cond)
        elif term.startswith("is:") or term.startswith("not:") or term[0] in "+-":
            if term[0] in "+-":
                cond = term[1:]
                lneg = term[0] == "-"
            else:
                cond = term[term.find(":") + 1:]
                lneg = term.startswith("not:")
            if cond == "complete":
                q = Q(is_complete=True)
            elif cond == "pull":
                q = Q(subject__contains='[PULL') | Q(subject__contains='[GIT PULL')
            elif cond == "reviewed":
                q = Q(properties__name="reviewed",
                      properties__value=True)
            elif cond in ("obsoleted", "old"):
                q = Q(properties__name="obsoleted-by",
                      properties__value__isnull=False) & \
                    ~Q(properties__name="obsoleted-by",
                      properties__value__iexact='')
            elif cond == "applied":
                q = Q(results__name="git",
                      results__status=Result.SUCCESS)
            elif cond == "tested":
                q = Q(properties__name="testing.done",
                      properties__value=True)
            elif cond == "merged":
                q = Q(is_merged=True)
            else:
                q = as_keywords(term)
            if lneg:
                neg = not neg
        elif term.startswith("has:"):
            cond = term[term.find(":") + 1:]
            if cond == "replies":
                q = Q(last_comment_date__isnull=False)
            else:
                q = Q(properties__name=cond)
        elif term.startswith("project:"):
            cond = term[term.find(":") + 1:]
            self._projects.add(cond)
            q = Q(project__name=cond) | Q(project__parent_project__name=cond)
        else:
            # Keyword in subject is the default
            q = as_keywords(term)
        if neg:
            return query.exclude(pk__in=query.filter(q))
        else:
            return query.filter(q)

    def last_keywords(self):
        return getattr(self, "_last_keywords", [])

    def project(self):
        return next(iter(self._projects)) if len(self._projects) == 1 else None

    def search_series(self, *terms, queryset=None):
        self._last_keywords = []
        self._projects = set()
        if queryset is None:
            queryset = Message.objects.series_heads()
        for t in terms:
            queryset = self._process_term(queryset, t)
        return queryset

    def _process_age_term(self, query, cond):
        import datetime
        def human_to_seconds(n, unit):
            if unit == "d":
                return n * 86400
            elif unit == "w":
                return n * 86400 * 7
            elif unit == "m":
                return n * 86400 * 30
            elif unit == "y":
                return n * 86400 * 365
            raise Exception("No unit specified")

        if cond.startswith("<"):
            less = True
            cond = cond[1:]
        elif cond.startswith(">"):
            less = False
            cond = cond[1:]
        else:
            less = False
        num, unit = cond[:-1], cond[-1].lower()
        if not num.isdigit() or not unit in "dwmy":
            raise InvalidSearchTerm("Invalid age string: %s" % cond)
        sec = human_to_seconds(int(num), unit)
        p = datetime.datetime.now() - datetime.timedelta(0, sec)
        if less:
            q = Q(date__gte=p)
        else:
            q = Q(date__lte=p)
        return q
<|endoftext|>