<commit_before>#!/usr/bin/env python3
# -*- coding: utf-8 -*-

import argparse

import os
import sys

import tensorflow as tf

from flask import Flask, request, jsonify
from flask.views import View

from inferbeddings.nli.util import SNLI, count_trainable_parameters, train_tokenizer_on_instances
from inferbeddings.nli import ConditionalBiLSTM, FeedForwardDAM, FeedForwardDAMP, ESIMv1

from werkzeug.serving import WSGIRequestHandler, BaseWSGIServer

import logging

logging.basicConfig(stream=sys.stdout, level=logging.DEBUG)
logger = logging.getLogger(os.path.basename(sys.argv[0]))

WSGIRequestHandler.protocol_version = "HTTP/1.1"
BaseWSGIServer.protocol_version = "HTTP/1.1"

app = Flask('nli-service')

class InvalidAPIUsage(Exception):
    """
    Class used for handling error messages.
    """
    DEFAULT_STATUS_CODE = 400

    def __init__(self, message, status_code=None, payload=None):
        Exception.__init__(self)
        self.message = message
        self.status_code = self.DEFAULT_STATUS_CODE
        if status_code is not None:
            self.status_code = status_code
        self.payload = payload

    def to_dict(self):
        rv = dict(self.payload or ())
        rv['message'] = self.message
        return rv

@app.errorhandler(InvalidAPIUsage)
def handle_invalid_usage(error):
    response = jsonify(error.to_dict())
    response.status_code = error.status_code
    return response

def main(argv):
    def formatter(prog):
        return argparse.HelpFormatter(prog, max_help_position=100, width=200)

    argparser = argparse.ArgumentParser('NLI Service', formatter_class=formatter)

    argparser.add_argument('--train', '-t', action='store', type=str, default='data/snli/snli_1.0_train.jsonl.gz')
    argparser.add_argument('--valid', '-v', action='store', type=str, default='data/snli/snli_1.0_dev.jsonl.gz')
    argparser.add_argument('--test', '-T', action='store', type=str, default='data/snli/snli_1.0_test.jsonl.gz')

    argparser.add_argument('--model', '-m', action='store', type=str, default='cbilstm',
                           choices=['cbilstm', 'ff-dam', 'ff-damp', 'esim1'])

    argparser.add_argument('--embedding-size', action='store', type=int, default=300)
    argparser.add_argument('--representation-size', action='store', type=int, default=200)
    argparser.add_argument('--hidden-size', action='store', type=int, default=200)

    argparser.add_argument('--semi-sort', action='store_true')
    argparser.add_argument('--fixed-embeddings', '-f', action='store_true')
    argparser.add_argument('--normalized-embeddings', '-n', action='store_true')
    argparser.add_argument('--use-masking', action='store_true')
    argparser.add_argument('--prepend-null-token', action='store_true')

    argparser.add_argument('--restore', '-r', action='store', type=str, default=None)

    args = argparser.parse_args(argv)

    train_path, valid_path, test_path = args.train, args.valid, args.test

    model_name = args.model

    embedding_size = args.embedding_size
    representation_size = args.representation_size

    is_fixed_embeddings = args.fixed_embeddings
    use_masking = args.use_masking
    prepend_null_token = args.prepend_null_token

    restore_path = args.restore

    logger.debug('Reading corpus ..')
    train_instances, dev_instances, test_instances = SNLI.generate(
        train_path=train_path, valid_path=valid_path, test_path=test_path)

    logger.info('Train size: {}\tDev size: {}\tTest size: {}'.format(len(train_instances), len(dev_instances), len(test_instances)))

    logger.debug('Parsing corpus ..')

    num_words = None
    all_instances = train_instances + dev_instances + test_instances
    qs_tokenizer, a_tokenizer = train_tokenizer_on_instances(all_instances, num_words=num_words)

    neutral_idx = a_tokenizer.word_index['neutral'] - 1
    entailment_idx = a_tokenizer.word_index['entailment'] - 1
    contradiction_idx = a_tokenizer.word_index['contradiction'] - 1

    vocab_size = qs_tokenizer.num_words if qs_tokenizer.num_words else len(qs_tokenizer.word_index) + 1

    sentence1_ph = tf.placeholder(dtype=tf.int32, shape=[None, None], name='sentence1')
    sentence2_ph = tf.placeholder(dtype=tf.int32, shape=[None, None], name='sentence2')

    sentence1_length_ph = tf.placeholder(dtype=tf.int32, shape=[None], name='sentence1_length')
    sentence2_length_ph = tf.placeholder(dtype=tf.int32, shape=[None], name='sentence2_length')

    embedding_layer = tf.get_variable('embeddings', shape=[vocab_size, embedding_size],
                                      initializer=tf.contrib.layers.xavier_initializer(),
                                      trainable=not is_fixed_embeddings)

    sentence1_embedding = tf.nn.embedding_lookup(embedding_layer, sentence1_ph)
    sentence2_embedding = tf.nn.embedding_lookup(embedding_layer, sentence2_ph)

    dropout_keep_prob_ph = tf.placeholder(tf.float32, name='dropout_keep_prob')

    model_kwargs = dict(
        sequence1=sentence1_embedding, sequence1_length=sentence1_length_ph,
        sequence2=sentence2_embedding, sequence2_length=sentence2_length_ph,
        representation_size=representation_size, dropout_keep_prob=dropout_keep_prob_ph)

    model_class = None
    if model_name == 'cbilstm':
        model_class = ConditionalBiLSTM
    elif model_name == 'ff-dam':
        ff_kwargs = dict(use_masking=use_masking, prepend_null_token=prepend_null_token)
        model_kwargs.update(ff_kwargs)
        model_class = FeedForwardDAM
    elif model_name == 'ff-damp':
        ff_kwargs = dict(use_masking=use_masking, prepend_null_token=prepend_null_token)
        model_kwargs.update(ff_kwargs)
        model_class = FeedForwardDAMP
    elif model_name == 'esim1':
        ff_kwargs = dict(use_masking=use_masking)
        model_kwargs.update(ff_kwargs)
        model_class = ESIMv1

    assert model_class is not None
    model = model_class(**model_kwargs)

    with tf.Session() as session:
        saver = tf.train.Saver()
        logger.debug('Total parameters: {}'.format(count_trainable_parameters()))
        saver.restore(session, restore_path)

        class Service(View):
            methods = ['GET', 'POST']

            def dispatch_request(self):
                sentence1 = request.form['sentence1'] if 'sentence1' in request.form else request.args.get('sentence1')
                sentence2 = request.form['sentence2'] if 'sentence2' in request.form else request.args.get('sentence2')

                sentence1_seq = [item for sublist in qs_tokenizer.texts_to_sequences([sentence1]) for item in sublist]
                sentence2_seq = [item for sublist in qs_tokenizer.texts_to_sequences([sentence2]) for item in sublist]

                sentence1_seq = sentence1_seq + ([0] * (48 - len(sentence1_seq)))
                sentence2_seq = sentence1_seq + ([0] * (50 - len(sentence2_seq)))

                # Compute answer
                feed_dict = {
                    sentence1_ph: [sentence1_seq], sentence2_ph: [sentence2_seq],
                    sentence1_length_ph: [len(sentence1_seq)], sentence2_length_ph: [len(sentence2_seq)],
                    dropout_keep_prob_ph: 1.0
                }

                predictions = session.run(tf.nn.softmax(model.logits), feed_dict=feed_dict)[0]
                answer = {
                    'neutral': str(predictions[neutral_idx]),
                    'contradiction': str(predictions[contradiction_idx]),
                    'entailment': str(predictions[entailment_idx])
                }

                return jsonify(answer)

        app.add_url_rule('/v1/nli', view_func=Service.as_view('request'))

        app.run(host='0.0.0.0', port=8889, debug=True)

if __name__ == '__main__':
    logging.basicConfig(level=logging.DEBUG)
    main(sys.argv[1:])
<commit_msg>better code<commit_after>#!/usr/bin/env python3
# -*- coding: utf-8 -*-

import argparse

import os
import sys

import tensorflow as tf

from flask import Flask, request, jsonify
from flask.views import View

from inferbeddings.nli.util import SNLI, count_trainable_parameters, train_tokenizer_on_instances
from inferbeddings.nli import ConditionalBiLSTM, FeedForwardDAM, FeedForwardDAMP, ESIMv1

from werkzeug.serving import WSGIRequestHandler, BaseWSGIServer

import logging

logging.basicConfig(stream=sys.stdout, level=logging.DEBUG)
logger = logging.getLogger(os.path.basename(sys.argv[0]))

WSGIRequestHandler.protocol_version = "HTTP/1.1"
BaseWSGIServer.protocol_version = "HTTP/1.1"

app = Flask('nli-service')

class InvalidAPIUsage(Exception):
    """
    Class used for handling error messages.
    """
    DEFAULT_STATUS_CODE = 400

    def __init__(self, message, status_code=None, payload=None):
        Exception.__init__(self)
        self.message = message
        self.status_code = self.DEFAULT_STATUS_CODE
        if status_code is not None:
            self.status_code = status_code
        self.payload = payload

    def to_dict(self):
        rv = dict(self.payload or ())
        rv['message'] = self.message
        return rv

@app.errorhandler(InvalidAPIUsage)
def handle_invalid_usage(error):
    response = jsonify(error.to_dict())
    response.status_code = error.status_code
    return response

def main(argv):
    def formatter(prog):
        return argparse.HelpFormatter(prog, max_help_position=100, width=200)

    argparser = argparse.ArgumentParser('NLI Service', formatter_class=formatter)

    argparser.add_argument('--train', '-t', action='store', type=str, default='data/snli/snli_1.0_train.jsonl.gz')
    argparser.add_argument('--valid', '-v', action='store', type=str, default='data/snli/snli_1.0_dev.jsonl.gz')
    argparser.add_argument('--test', '-T', action='store', type=str, default='data/snli/snli_1.0_test.jsonl.gz')

    argparser.add_argument('--model', '-m', action='store', type=str, default='cbilstm',
                           choices=['cbilstm', 'ff-dam', 'ff-damp', 'esim1'])

    argparser.add_argument('--embedding-size', action='store', type=int, default=300)
    argparser.add_argument('--representation-size', action='store', type=int, default=200)
    argparser.add_argument('--hidden-size', action='store', type=int, default=200)

    argparser.add_argument('--semi-sort', action='store_true')
    argparser.add_argument('--fixed-embeddings', '-f', action='store_true')
    argparser.add_argument('--normalized-embeddings', '-n', action='store_true')
    argparser.add_argument('--use-masking', action='store_true')
    argparser.add_argument('--prepend-null-token', action='store_true')

    argparser.add_argument('--restore', '-r', action='store', type=str, default=None)

    args = argparser.parse_args(argv)

    train_path, valid_path, test_path = args.train, args.valid, args.test

    model_name = args.model

    embedding_size = args.embedding_size
    representation_size = args.representation_size

    is_fixed_embeddings = args.fixed_embeddings
    use_masking = args.use_masking
    prepend_null_token = args.prepend_null_token

    restore_path = args.restore

    logger.debug('Reading corpus ..')
    train_instances, dev_instances, test_instances = SNLI.generate(
        train_path=train_path, valid_path=valid_path, test_path=test_path)

    logger.info('Train size: {}\tDev size: {}\tTest size: {}'.format(len(train_instances), len(dev_instances), len(test_instances)))

    logger.debug('Parsing corpus ..')

    num_words = None
    all_instances = train_instances + dev_instances + test_instances
    qs_tokenizer, a_tokenizer = train_tokenizer_on_instances(all_instances, num_words=num_words)

    neutral_idx = a_tokenizer.word_index['neutral'] - 1
    entailment_idx = a_tokenizer.word_index['entailment'] - 1
    contradiction_idx = a_tokenizer.word_index['contradiction'] - 1

    vocab_size = qs_tokenizer.num_words if qs_tokenizer.num_words else len(qs_tokenizer.word_index) + 1

    sentence1_ph = tf.placeholder(dtype=tf.int32, shape=[None, None], name='sentence1')
    sentence2_ph = tf.placeholder(dtype=tf.int32, shape=[None, None], name='sentence2')

    sentence1_length_ph = tf.placeholder(dtype=tf.int32, shape=[None], name='sentence1_length')
    sentence2_length_ph = tf.placeholder(dtype=tf.int32, shape=[None], name='sentence2_length')

    embedding_layer = tf.get_variable('embeddings', shape=[vocab_size, embedding_size],
                                      initializer=tf.contrib.layers.xavier_initializer(),
                                      trainable=not is_fixed_embeddings)

    sentence1_embedding = tf.nn.embedding_lookup(embedding_layer, sentence1_ph)
    sentence2_embedding = tf.nn.embedding_lookup(embedding_layer, sentence2_ph)

    dropout_keep_prob_ph = tf.placeholder(tf.float32, name='dropout_keep_prob')

    model_kwargs = dict(
        sequence1=sentence1_embedding, sequence1_length=sentence1_length_ph,
        sequence2=sentence2_embedding, sequence2_length=sentence2_length_ph,
        representation_size=representation_size, dropout_keep_prob=dropout_keep_prob_ph)

    model_class = None
    if model_name == 'cbilstm':
        model_class = ConditionalBiLSTM
    elif model_name == 'ff-dam':
        ff_kwargs = dict(use_masking=use_masking, prepend_null_token=prepend_null_token)
        model_kwargs.update(ff_kwargs)
        model_class = FeedForwardDAM
    elif model_name == 'ff-damp':
        ff_kwargs = dict(use_masking=use_masking, prepend_null_token=prepend_null_token)
        model_kwargs.update(ff_kwargs)
        model_class = FeedForwardDAMP
    elif model_name == 'esim1':
        ff_kwargs = dict(use_masking=use_masking)
        model_kwargs.update(ff_kwargs)
        model_class = ESIMv1

    assert model_class is not None
    model = model_class(**model_kwargs)

    with tf.Session() as session:
        saver = tf.train.Saver()
        logger.debug('Total parameters: {}'.format(count_trainable_parameters()))
        saver.restore(session, restore_path)

        class Service(View):
            methods = ['GET', 'POST']

            def dispatch_request(self):
                sentence1 = request.form['sentence1'] if 'sentence1' in request.form else request.args.get('sentence1')
                sentence2 = request.form['sentence2'] if 'sentence2' in request.form else request.args.get('sentence2')

                sentence1_seq = [item for sublist in qs_tokenizer.texts_to_sequences([sentence1]) for item in sublist]
                sentence2_seq = [item for sublist in qs_tokenizer.texts_to_sequences([sentence2]) for item in sublist]

                padded_sentence1_seq = sentence1_seq + ([0] * (48 - len(sentence1_seq)))
                padded_sentence2_seq = sentence2_seq + ([0] * (50 - len(sentence2_seq)))

                # Compute answer
                feed_dict = {
                    sentence1_ph: [padded_sentence1_seq],
                    sentence2_ph: [padded_sentence2_seq],
                    sentence1_length_ph: [len(sentence1_seq)],
                    sentence2_length_ph: [len(sentence2_seq)],
                    dropout_keep_prob_ph: 1.0
                }

                predictions = session.run(tf.nn.softmax(model.logits), feed_dict=feed_dict)[0]
                answer = {
                    'neutral': str(predictions[neutral_idx]),
                    'contradiction': str(predictions[contradiction_idx]),
                    'entailment': str(predictions[entailment_idx])
                }

                return jsonify(answer)

        app.add_url_rule('/v1/nli', view_func=Service.as_view('request'))

        app.run(host='0.0.0.0', port=8889, debug=True)

if __name__ == '__main__':
    logging.basicConfig(level=logging.DEBUG)
    main(sys.argv[1:])
<|endoftext|>


<commit_before># ##########################################
# Version 0.1
# author: Brian Torres-Gil
#
# About this script:
# This script retrieves an xml file with new apps so they
# can be highlighted in a dashboard.
#
# Script's actions and warning messages are logged in $SPLUNK_HOME/var/log/splunk/python.log
############################################
###########################################
# if you DO want to go through a proxy, e.g., HTTP_PROXY={squid:'2.2.2.2'}
HTTP_PROXY = {}
DEBUG = False
#########################################################
# Do NOT modify anything below this line unless you are
# certain of the ramifications of the changes
#########################################################
import splunk.Intersplunk  # so you can interact with Splunk
import splunk.entity as entity  # for splunk config info
import splunk.mining.dcutils as dcu
import urllib  # for urllib.urlencode()
import urllib2  # make http requests to PAN firewall
import sys  # for system params and sys.exit()
import traceback
import xml.etree.ElementTree as ET  # for xml parsing

def createOpener():
    """Create a generic opener for http
    This is particularly helpful when there is a proxy server in line"""
    # Thanks to: http://www.decalage.info/en/python/urllib2noproxy
    proxy_handler = urllib2.ProxyHandler(HTTP_PROXY)
    opener = urllib2.build_opener(proxy_handler)
    urllib2.install_opener(opener)
    return opener

def retrieveNewApps():
    # Create a urllib2 opener
    opener = createOpener()
    # URL for WildFire cloud API
    newAppUrl = 'http://ww2.paloaltonetworks.com/iphone/NewApps.aspx'
    # Create a request object
    newAppReq = urllib2.Request(newAppUrl)
    # Make the request
    result = opener.open(newAppReq)
    return result

# an empty dictionary will be used to hold system values
settings = dict()
# results contains the data from the search results and settings contains the sessionKey that we can use to talk to splunk
results, unused1, settings = splunk.Intersplunk.getOrganizedResults()
args, kwargs = splunk.Intersplunk.getKeywordsAndOptions()
#logger.debug(settings) #For debugging
# get the sessionKey
sessionKey = settings['sessionKey']

try:
    if 'debug' in kwargs:
        DEBUG = kwargs['debug']
    # setup the logger. $SPLUNK_HOME/var/log/splunk/python.log
    logger = dcu.getLogger().getChild('retrieveNewApps')

    if DEBUG:
        logger.setLevel(DEBUG)

    existing_apps = []
    for app in results:
        existing_apps.append(str(app['app{@name}']))

    results = []
    logger.debug("Existing apps already known and considered: %s" % (len(existing_apps),))
    logger.debug(existing_apps)

    logger.debug("Getting new Apps from Palo Alto Networks")
    resp = retrieveNewApps()
    logger.debug("Apps retrieved")
    xml = resp.read()

    logger.debug("Apps read")
    xmlroot = ET.fromstring(xml)
    logger.debug("Apps parsed")

    newapps = xmlroot.findall("./entry")
    logger.debug("Found %s new apps at Palo Alto Networks" % (len(newapps),))

    for app in newapps:
        app.tag = 'app'
        if app.get('name') not in existing_apps:
            results.append({"_raw": ET.tostring(app)})

    logger.debug("Found %s new apps that weren't already known" % (len(results),))

    # output the complete results sent back to splunk
    splunk.Intersplunk.outputResults(results)

except Exception, e:
    stack = traceback.format_exc()
    logger.warn("Exception:")
    logger.warn(stack)
    raise Exception("Exception while getting new apps. Error: %s" % (str(e),))

<commit_msg>Update retrieveNewApps script to use new common methods<commit_after>#!/usr/bin/env python

# Copyright (c) 2015, Palo Alto Networks
#
# Permission to use, copy, modify, and/or distribute this software for any
# purpose with or without fee is hereby granted, provided that the above
# copyright notice and this permission notice appear in all copies.
#
# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.

# Author: Brian Torres-Gil <btorres-gil@paloaltonetworks.com>

"""Retrieve the latest additions to Applipedia

About this script
-----------------
This script retrieves an xml file with new apps so they
can be highlighted in a dashboard.

As with other scripts in this app, all script actions and warning
messages are logged in $SPLUNK_HOME/var/log/splunk/python.log
"""

# if you DO want to go through a proxy, e.g., HTTP_PROXY={squid:'2.2.2.2'}
HTTP_PROXY = {}

#########################################################
# Do NOT modify anything below this line unless you are
# certain of the ramifications of the changes
#########################################################

import sys  # for system params and sys.exit()
import os
import xml.etree.ElementTree as ET  # for xml parsing
import urllib2  # make http requests to PAN firewall

libpath = os.path.dirname(os.path.abspath(__file__))
sys.path[:0] = [os.path.join(libpath, 'lib')]
import common

logger = common.logging.getLogger().getChild('panRetrieveNewApps')

try:
    import splunk.Intersplunk  # so you can interact with Splunk
    import splunk.entity as entity  # for splunk config info

    libpath = os.path.dirname(os.path.abspath(__file__))
    sys.path[:0] = [os.path.join(libpath, 'lib')]
    sys.path[:0] = [os.path.join(libpath, 'lib', 'pan-python', 'lib')]
    sys.path[:0] = [os.path.join(libpath, 'lib', 'pandevice')]
    import pandevice
    from pandevice.panorama import Panorama
    from pandevice.firewall import Firewall
    import pan.xapi

    from common import log

except Exception as e:
    # Handle exception to produce logs to python.log
    common.exit_with_error(e)

def createOpener():
    """Create a generic opener for http
    This is particularly helpful when there is a proxy server in line"""
    # Thanks to: http://www.decalage.info/en/python/urllib2noproxy
    proxy_handler = urllib2.ProxyHandler(HTTP_PROXY)
    opener = urllib2.build_opener(proxy_handler)
    urllib2.install_opener(opener)
    return opener

def retrieveNewApps():
    # Create a urllib2 opener
    opener = createOpener()
    # URL for WildFire cloud API
    newAppUrl = 'http://ww2.paloaltonetworks.com/iphone/NewApps.aspx'
    # Create a request object
    newAppReq = urllib2.Request(newAppUrl)
    # Make the request
    result = opener.open(newAppReq)
    return result

try:
    # Get arguments
    args, kwargs = splunk.Intersplunk.getKeywordsAndOptions()

    # Enable debugging by passing 'debug=yes' as an argument of
    # the command on the Splunk searchbar.

    debug = common.check_debug(kwargs)

    # Results contains the data from the search results
    results, unused1, settings = splunk.Intersplunk.getOrganizedResults()

    existing_apps = []
    for app in results:
        existing_apps.append(str(app['app{@name}']))

    results = []
    log(debug, "Existing apps already known and considered: %s" % (len(existing_apps),))
    log(debug, existing_apps)

    log(debug, "Getting new Apps from Palo Alto Networks")
    resp = retrieveNewApps()
    log(debug, "Apps retrieved")
    xml = resp.read()

    log(debug, "Apps read")
    xmlroot = ET.fromstring(xml)
    log(debug, "Apps parsed")

    newapps = xmlroot.findall("./entry")
    log(debug, "Found %s new apps at Palo Alto Networks" % (len(newapps),))

    for app in newapps:
        app.tag = 'app'
        if app.get('name') not in existing_apps:
            results.append({"_raw": ET.tostring(app)})

    log(debug, "Found %s new apps that weren't already known" % (len(results),))

    # output the complete results sent back to splunk
    splunk.Intersplunk.outputResults(results)

except Exception, e:
    common.exit_with_error("Exception while getting new apps. Error: %s" % str(e))

<|endoftext|>


<commit_before># Copyright (c) 2016 Uber Technologies, Inc.
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.

from __future__ import absolute_import

import logging
import sys
from collections import namedtuple
import six

import tornado
import tornado.gen
from tornado import gen
from tornado.iostream import StreamClosedError

from tchannel.request import Request
from tchannel.request import TransportHeaders
from tchannel.response import response_from_mixed
from ..errors import BadRequestError
from ..errors import UnexpectedError
from ..errors import TChannelError
from ..event import EventType
from ..messages import Types
from ..serializer.raw import RawSerializer
from .response import Response as DeprecatedResponse
from .. import tracing

log = logging.getLogger('tchannel')

Handler = namedtuple('Handler', 'endpoint req_serializer resp_serializer')

class RequestDispatcher(object):
    """A synchronous RequestHandler that dispatches calls to different
    endpoints based on ``arg1``.

    Endpoints are registered using ``register`` or the ``route``
    decorator.

    .. code-block:: python

        handler = # ...

        @handler.route('my_method')
        def my_method(request, response):
            response.write('hello world')
    """

    FALLBACK = object()

    def __init__(self, _handler_returns_response=False):
        self.handlers = {}
        self.register(self.FALLBACK, self.not_found)
        self._handler_returns_response = _handler_returns_response

    _HANDLER_NAMES = {
        Types.CALL_REQ: 'pre_call',
        Types.CALL_REQ_CONTINUE: 'pre_call'
    }

    def handle(self, message, connection):
        # TODO assert that the handshake was already completed
        assert message, "message must not be None"

        if message.message_type not in self._HANDLER_NAMES:
            # TODO handle this more gracefully
            raise NotImplementedError("Unexpected message: %s" % str(message))

        handler_name = "handle_" + self._HANDLER_NAMES[message.message_type]
        return getattr(self, handler_name)(message, connection)

    def handle_pre_call(self, message, connection):
        """Handle incoming request message including CallRequestMessage and
        CallRequestContinueMessage

        This method will build the User friendly request object based on the
        incoming messages.

        It passes all the messages into the message_factory to build the init
        request object. Only when it get a CallRequestMessage and a completed
        arg_1=argstream[0], the message_factory will return a request object.
        Then it will trigger the async handle_call method.

        :param message: CallRequestMessage or CallRequestContinueMessage
        :param connection: tornado connection
        """
        req = None
        try:
            req = connection.request_message_factory.build(message)
            # message_factory will create Request only when it receives
            # CallRequestMessage. It will return None, if it receives
            # CallRequestContinueMessage.
            if req:
                self.handle_call(req, connection)

        except TChannelError as e:
            log.warn('Received a bad request.', exc_info=True)
            if req:
                e.tracing = req.tracing
            connection.send_error(e)

    @tornado.gen.coroutine
    def handle_call(self, request, connection):
        # read arg_1 so that handle_call is able to get the endpoint
        # name and find the endpoint handler.
        # the arg_1 value will be store in the request.endpoint field.

        # NOTE: after here, the correct way to access value of arg_1 is through
        # request.endpoint. The original argstream[0] is no longer valid. If
        # user still tries read from it, it will return empty.
        chunk = yield request.argstreams[0].read()
        while chunk:
            if six.PY3:
                if isinstance(chunk, str) and \
                        isinstance(request.endpoint, bytes) or \
                        isinstance(request.endpoint, str) and \
                        isinstance(chunk, bytes):
                    chunk = chunk.decode('utf8')

            request.endpoint += chunk
            chunk = yield request.argstreams[0].read()

        log.debug('Received a call to %s.', request.endpoint)

        tchannel = connection.tchannel

        yield tchannel.event_emitter.fire(
            EventType.before_receive_request, request)

        handler = self.get_endpoint(request.endpoint)

        requested_as = request.headers.get('as', None)
        expected_as = handler.req_serializer.name

        if request.endpoint in self.handlers and requested_as != expected_as:
            connection.send_error(BadRequestError(
                description=(
                    "Server expected a '%s' but request is '%s'"
                    % (
                        expected_as,
                        requested_as,
                    )
                ),
                id=request.id,
                tracing=request.tracing,
            ))

            raise gen.Return(None)

        request.serializer = handler.req_serializer
        response = DeprecatedResponse(
            id=request.id,
            checksum=request.checksum,
            tracing=request.tracing,
            connection=connection,
            headers={'as': request.headers.get('as', 'raw')},
            serializer=handler.resp_serializer,
        )

        def _on_post_response(future):
            if not future.exception():
                return

            # Failed to write response because client disappeared. Nothing to
            # do.
            if isinstance(future.exception(), StreamClosedError):
                return

            log.error('failed to write response', exc_info=future.exc_info())

        connection.post_response(response).add_done_callback(_on_post_response)

        tracer = tracing.ServerTracer(
            tracer=tchannel.tracer, operation_name=request.endpoint
        )
        tracer.start_basic_span(request)

        try:
            # New impl - the handler takes a request and returns a response
            if self._handler_returns_response:
                # convert deprecated req to new top-level req
                b = yield request.get_body()
                he = yield request.get_header()
                t = TransportHeaders.from_dict(request.headers)
                new_req = Request(
                    body=b,
                    headers=he,
                    transport=t,
                    endpoint=request.endpoint,
                    service=request.service,
                    timeout=request.ttl,
                )
                with tracer.start_span(
                    request=request, headers=he,
                    peer_host=connection.remote_host,
                    peer_port=connection.remote_host_port
                ) as span:
                    context_provider = tchannel.context_provider_fn()
                    with context_provider.span_in_context(span):
                        # Cannot yield while inside the StackContext
                        f = handler.endpoint(new_req)
                    new_resp = yield gen.maybe_future(f)

                # instantiate a tchannel.Response
                new_resp = response_from_mixed(new_resp)

                response.code = new_resp.status

                # assign resp values to dep response
                response.write_header(new_resp.headers)

                if new_resp.body is not None:
                    response.write_body(new_resp.body)

            # Dep impl - the handler is provided with a req & resp writer
            else:
                with tracer.start_span(
                    request=request, headers={},
                    peer_host=connection.remote_host,
                    peer_port=connection.remote_host_port
                ) as span:
                    context_provider = tchannel.context_provider_fn()
                    with context_provider.span_in_context(span):
                        # Cannot yield while inside the StackContext
                        f = handler.endpoint(request, response)

                    yield gen.maybe_future(f)

            response.flush()
        except TChannelError as e:
            e.tracing = request.tracing
            e.id = request.id
            connection.send_error(e)
        except Exception as e:
            # Maintain a reference to our original exc info because we stomp
            # the traceback below.
            exc_info = sys.exc_info()
            exc_type, exc_obj, exc_tb = exc_info
            try:
                # Walk to the traceback to find our offending line.
                while exc_tb.tb_next is not None:
                    exc_tb = exc_tb.tb_next

                description = "%r from %s in %s:%s" % (
                    e,
                    request.endpoint,
                    exc_tb.tb_frame.f_code.co_filename,
                    exc_tb.tb_lineno,
                )
                error = UnexpectedError(
                    description=description,
                    id=request.id,
                    tracing=request.tracing,
                )

                response.set_exception(error, exc_info=exc_info)
                connection.request_message_factory.remove_buffer(response.id)

                connection.send_error(error)
                yield tchannel.event_emitter.fire(
                    EventType.on_exception,
                    request,
                    error,
                )
                log.error("Unexpected error", exc_info=exc_info)
            finally:
                # Clean up circular reference.
                # https://docs.python.org/2/library/sys.html#sys.exc_info
                del exc_tb
                del exc_info
        raise gen.Return(response)

    def get_endpoint(self, name):
        handler = self.handlers.get(name)

        if handler is None:
            handler = self.handlers[self.FALLBACK]

        return handler

    def register(
            self,
            rule,
            handler,
            req_serializer=None,
            resp_serializer=None
    ):
        """Register a new endpoint with the given name.

        .. code-block:: python

            @dispatcher.register('is_healthy')
            def check_health(request, response):
                # ...

        :param rule:
            Name of the endpoint. Incoming Call Requests must have this as
            ``arg1`` to dispatch to this handler.

            If ``RequestHandler.FALLBACK`` is specified as a rule, the given
            handler will be used as the 'fallback' handler when requests don't
            match any registered rules.

        :param handler:
            A function that gets called with ``Request`` and ``Response``.

        :param req_serializer:
            Arg scheme serializer of this endpoint. It should be
            ``RawSerializer``, ``JsonSerializer``, and ``ThriftSerializer``.

        :param resp_serializer:
            Arg scheme serializer of this endpoint. It should be
            ``RawSerializer``, ``JsonSerializer``, and ``ThriftSerializer``.
        """

        assert handler, "handler must not be None"
        req_serializer = req_serializer or RawSerializer()
        resp_serializer = resp_serializer or RawSerializer()
        self.handlers[rule] = Handler(handler, req_serializer, resp_serializer)

    @staticmethod
    def not_found(request, response=None):
        """Default behavior for requests to unrecognized endpoints."""
        raise BadRequestError(
            description="Endpoint '%s' is not defined" % (
                request.endpoint,
            ),
        )
<commit_msg>Clean up some dirty logic<commit_after># Copyright (c) 2016 Uber Technologies, Inc.
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.

from __future__ import absolute_import

import logging
import sys
from collections import namedtuple
import six

import tornado
import tornado.gen
from tornado import gen
from tornado.iostream import StreamClosedError

from tchannel.request import Request
from tchannel.request import TransportHeaders
from tchannel.response import response_from_mixed
from ..errors import BadRequestError
from ..errors import UnexpectedError
from ..errors import TChannelError
from ..event import EventType
from ..messages import Types
from ..serializer.raw import RawSerializer
from .response import Response as DeprecatedResponse
from .. import tracing

log = logging.getLogger('tchannel')

Handler = namedtuple('Handler', 'endpoint req_serializer resp_serializer')

class RequestDispatcher(object):
    """A synchronous RequestHandler that dispatches calls to different
    endpoints based on ``arg1``.

    Endpoints are registered using ``register`` or the ``route``
    decorator.

    .. code-block:: python

        handler = # ...

        @handler.route('my_method')
        def my_method(request, response):
            response.write('hello world')
    """

    FALLBACK = object()

    def __init__(self, _handler_returns_response=False):
        self.handlers = {}
        self.register(self.FALLBACK, self.not_found)
        self._handler_returns_response = _handler_returns_response

    _HANDLER_NAMES = {
        Types.CALL_REQ: 'pre_call',
        Types.CALL_REQ_CONTINUE: 'pre_call'
    }

    def handle(self, message, connection):
        # TODO assert that the handshake was already completed
        assert message, "message must not be None"

        if message.message_type not in self._HANDLER_NAMES:
            # TODO handle this more gracefully
            raise NotImplementedError("Unexpected message: %s" % str(message))

        handler_name = "handle_" + self._HANDLER_NAMES[message.message_type]
        return getattr(self, handler_name)(message, connection)

    def handle_pre_call(self, message, connection):
        """Handle incoming request message including CallRequestMessage and
        CallRequestContinueMessage

        This method will build the User friendly request object based on the
        incoming messages.

        It passes all the messages into the message_factory to build the init
        request object. Only when it get a CallRequestMessage and a completed
        arg_1=argstream[0], the message_factory will return a request object.
        Then it will trigger the async handle_call method.

        :param message: CallRequestMessage or CallRequestContinueMessage
        :param connection: tornado connection
        """
        req = None
        try:
            req = connection.request_message_factory.build(message)
            # message_factory will create Request only when it receives
            # CallRequestMessage. It will return None, if it receives
            # CallRequestContinueMessage.
            if req:
                self.handle_call(req, connection)

        except TChannelError as e:
            log.warn('Received a bad request.', exc_info=True)
            if req:
                e.tracing = req.tracing
            connection.send_error(e)

    @tornado.gen.coroutine
    def handle_call(self, request, connection):
        # read arg_1 so that handle_call is able to get the endpoint
        # name and find the endpoint handler.
        # the arg_1 value will be store in the request.endpoint field.

        # NOTE: after here, the correct way to access value of arg_1 is through
        # request.endpoint. The original argstream[0] is no longer valid. If
        # user still tries read from it, it will return empty.
        chunk = yield request.argstreams[0].read()
        while chunk:
            if six.PY3:
                if isinstance(request.endpoint, str) and \
                        isinstance(chunk, bytes):
                    chunk = chunk.decode('utf8')

            request.endpoint += chunk
            chunk = yield request.argstreams[0].read()

        log.debug('Received a call to %s.', request.endpoint)

        tchannel = connection.tchannel

        yield tchannel.event_emitter.fire(
            EventType.before_receive_request, request)

        handler = self.get_endpoint(request.endpoint)

        requested_as = request.headers.get('as', None)
        expected_as = handler.req_serializer.name

        if request.endpoint in self.handlers and requested_as != expected_as:
            connection.send_error(BadRequestError(
                description=(
                    "Server expected a '%s' but request is '%s'"
                    % (
                        expected_as,
                        requested_as,
                    )
                ),
                id=request.id,
                tracing=request.tracing,
            ))

            raise gen.Return(None)

        request.serializer = handler.req_serializer
        response = DeprecatedResponse(
            id=request.id,
            checksum=request.checksum,
            tracing=request.tracing,
            connection=connection,
            headers={'as': request.headers.get('as', 'raw')},
            serializer=handler.resp_serializer,
        )

        def _on_post_response(future):
            if not future.exception():
                return

            # Failed to write response because client disappeared. Nothing to
            # do.
            if isinstance(future.exception(), StreamClosedError):
                return

            log.error('failed to write response', exc_info=future.exc_info())

        connection.post_response(response).add_done_callback(_on_post_response)

        tracer = tracing.ServerTracer(
            tracer=tchannel.tracer, operation_name=request.endpoint
        )
        tracer.start_basic_span(request)

        try:
            # New impl - the handler takes a request and returns a response
            if self._handler_returns_response:
                # convert deprecated req to new top-level req
                b = yield request.get_body()
                he = yield request.get_header()
                t = TransportHeaders.from_dict(request.headers)
                new_req = Request(
                    body=b,
                    headers=he,
                    transport=t,
                    endpoint=request.endpoint,
                    service=request.service,
                    timeout=request.ttl,
                )
                with tracer.start_span(
                    request=request, headers=he,
                    peer_host=connection.remote_host,
                    peer_port=connection.remote_host_port
                ) as span:
                    context_provider = tchannel.context_provider_fn()
                    with context_provider.span_in_context(span):
                        # Cannot yield while inside the StackContext
                        f = handler.endpoint(new_req)
                    new_resp = yield gen.maybe_future(f)

                # instantiate a tchannel.Response
                new_resp = response_from_mixed(new_resp)

                response.code = new_resp.status

                # assign resp values to dep response
                response.write_header(new_resp.headers)

                if new_resp.body is not None:
                    response.write_body(new_resp.body)

            # Dep impl - the handler is provided with a req & resp writer
            else:
                with tracer.start_span(
                    request=request, headers={},
                    peer_host=connection.remote_host,
                    peer_port=connection.remote_host_port
                ) as span:
                    context_provider = tchannel.context_provider_fn()
                    with context_provider.span_in_context(span):
                        # Cannot yield while inside the StackContext
                        f = handler.endpoint(request, response)

                    yield gen.maybe_future(f)

            response.flush()
        except TChannelError as e:
            e.tracing = request.tracing
            e.id = request.id
            connection.send_error(e)
        except Exception as e:
            # Maintain a reference to our original exc info because we stomp
            # the traceback below.
            exc_info = sys.exc_info()
            exc_type, exc_obj, exc_tb = exc_info
            try:
                # Walk to the traceback to find our offending line.
                while exc_tb.tb_next is not None:
                    exc_tb = exc_tb.tb_next

                description = "%r from %s in %s:%s" % (
                    e,
                    request.endpoint,
                    exc_tb.tb_frame.f_code.co_filename,
                    exc_tb.tb_lineno,
                )
                error = UnexpectedError(
                    description=description,
                    id=request.id,
                    tracing=request.tracing,
                )

                response.set_exception(error, exc_info=exc_info)
                connection.request_message_factory.remove_buffer(response.id)

                connection.send_error(error)
                yield tchannel.event_emitter.fire(
                    EventType.on_exception,
                    request,
                    error,
                )
                log.error("Unexpected error", exc_info=exc_info)
            finally:
                # Clean up circular reference.
                # https://docs.python.org/2/library/sys.html#sys.exc_info
                del exc_tb
                del exc_info
        raise gen.Return(response)

    def get_endpoint(self, name):
        handler = self.handlers.get(name)

        if handler is None:
            handler = self.handlers[self.FALLBACK]

        return handler

    def register(
            self,
            rule,
            handler,
            req_serializer=None,
            resp_serializer=None
    ):
        """Register a new endpoint with the given name.

        .. code-block:: python

            @dispatcher.register('is_healthy')
            def check_health(request, response):
                # ...

        :param rule:
            Name of the endpoint. Incoming Call Requests must have this as
            ``arg1`` to dispatch to this handler.

            If ``RequestHandler.FALLBACK`` is specified as a rule, the given
            handler will be used as the 'fallback' handler when requests don't
            match any registered rules.

        :param handler:
            A function that gets called with ``Request`` and ``Response``.

        :param req_serializer:
            Arg scheme serializer of this endpoint. It should be
            ``RawSerializer``, ``JsonSerializer``, and ``ThriftSerializer``.

        :param resp_serializer:
            Arg scheme serializer of this endpoint. It should be
            ``RawSerializer``, ``JsonSerializer``, and ``ThriftSerializer``.
        """

        assert handler, "handler must not be None"
        req_serializer = req_serializer or RawSerializer()
        resp_serializer = resp_serializer or RawSerializer()
        self.handlers[rule] = Handler(handler, req_serializer, resp_serializer)

    @staticmethod
    def not_found(request, response=None):
        """Default behavior for requests to unrecognized endpoints."""
        raise BadRequestError(
            description="Endpoint '%s' is not defined" % (
                request.endpoint,
            ),
        )
<|endoftext|>


<commit_before>#!/usr/bin/python3

# requirements: something to probe

from sys import exit
from storage import *
from storageitu import *

class MyProbeCallbacks(ProbeCallbacksV3):

    def __init__(self):
        super(MyProbeCallbacks, self).__init__()

    def begin(self):
        print("begin callback")

    def end(self):
        print("end callback")

    def message(self, message):
        print("message callback")
        print(message)

    def error(self, message, what):
        print("error callback")
        print(message)
        print(what)
        return True

    def missing_command(self, message, what, command, used_features):
        print("missing command callback")
        print(message)
        print(what)
        print(command)
        print(used_features)
        return False

set_logger(get_logfile_logger())

my_probe_callbacks = MyProbeCallbacks()

environment = Environment(True)

try:
    storage = Storage(environment)
except LockException as exception:
    print(exception.what())
    print("locker pid %d" % exception.get_locker_pid())
    exit(1)

try:
    storage.probe(my_probe_callbacks)
except Exception as exception:
    print("caught exception")
    print(exception.what())
    exit(1)

probed = storage.get_probed()

print(probed)

probed.save("devicegraph.xml")

<commit_msg>- extended integration test<commit_after>#!/usr/bin/python3

# requirements: something to probe

from sys import exit
from storage import *
from storageitu import *

class MyProbeCallbacks(ProbeCallbacksV3):

    def __init__(self):
        super(MyProbeCallbacks, self).__init__()

    def begin(self):
        print("begin callback")

    def end(self):
        print("end callback")

    def message(self, message):
        print("message callback")
        print(message)

    def error(self, message, what):
        print("error callback")
        print(message)
        print(what)
        return True

    def missing_command(self, message, what, command, used_features):
        print("missing command callback")
        print(message)
        print(what)
        print(command)
        print(used_features)
        return False

set_logger(get_logfile_logger())

my_probe_callbacks = MyProbeCallbacks()

environment = Environment(True)

try:
    storage = Storage(environment)
except LockException as exception:
    print(exception.what())
    print("locker pid %d" % exception.get_locker_pid())
    exit(1)

try:
    storage.probe(my_probe_callbacks)
except Exception as exception:
    print("caught exception")
    print(exception.what())
    exit(1)

probed = storage.get_probed()

print(probed)

used_features = probed.used_features()
for i in range(64):
    if used_features & 1 << i:
        print(i, '', end='')
print()

probed.save("devicegraph.xml")
<|endoftext|>


<commit_before>import os
import base64
from datetime import datetime
from planetstack.config import Config
from util.logger import Logger, logging
from observer.steps import *
from django.db.models import F, Q

logger = Logger(level=logging.INFO)

class FailedDependency(Exception):
    pass

class SyncStep:
    """ A PlanetStack Sync step.

    Attributes:
        psmodel        Model name the step synchronizes
        dependencies    list of names of models that must be synchronized first if the current model depends on them
    """
    slow=False
    def get_prop(prop):
        try:
            sync_config_dir = Config().sync_config_dir
        except:
            sync_config_dir = '/etc/planetstack/sync'
        prop_config_path = '/'.join(sync_config_dir,self.name,prop)
        return open(prop_config_path).read().rstrip()

    def __init__(self, **args):
        """Initialize a sync step
           Keyword arguments:
                   name -- Name of the step
                provides -- PlanetStack models sync'd by this step
        """
        dependencies = []
        self.driver = args.get('driver')
        self.error_map = args.get('error_map')

        try:
            self.soft_deadline = int(self.get_prop('soft_deadline_seconds'))
        except:
            self.soft_deadline = 5 # 5 seconds

        return

    def fetch_pending(self, deletion=False):
        # This is the most common implementation of fetch_pending
        # Steps should override it if they have their own logic
        # for figuring out what objects are outstanding.
        main_obj = self.provides[0]
        if (not deletion):
            objs = main_obj.objects.filter(Q(enacted__lt=F('updated')) | Q(enacted=None))
        else:
            objs = main_obj.deleted_objects.all()

        return objs
        #return Sliver.objects.filter(ip=None)

    def check_dependencies(self, obj, failed):
        for dep in self.dependencies:
            peer_name = dep[0].lower() + dep[1:]    # django names are camelCased with the first letter lower
            try:
                peer_object = getattr(obj, peer_name)
            except:
                peer_object = None

            if (peer_object and peer_object.pk==failed.pk and type(peer_object)==type(failed)):
                raise FailedDependency("Failed dependency for %s:%s peer %s:%s failed  %s:%s" % (obj.__class__.__name__, str(obj.pk\
), peer_object.__class__.__name__, str(peer_object.pk), failed.__class__.__name__, str(failed.pk)))

    def call(self, failed=[], deletion=False):
        pending = self.fetch_pending(deletion)
        for o in pending:
            try:
                for f in failed:
                    self.check_dependencies(o,f) # Raises exception if failed
                if (deletion):
                    self.delete_record(o)
                    o.delete(purge=True)
                else:
                    self.sync_record(o)
                    o.enacted = datetime.now() # Is this the same timezone? XXX
                    o.backend_status = "OK"
                    o.save(update_fields=['enacted'])
            except Exception,e:
                try:
                    o.backend_status = self.error_map.map(str(e))
                except:
                    o.backend_status = str(e)

                if (o.pk):
                    o.save(update_fields=['backend_status'])

                logger.log_exc("sync step failed!")
                failed.append(o)

        return failed

    def __call__(self, **args):
        return self.call(**args)
<commit_msg>Propagate backend_status to failed dependent objects<commit_after>import os
import base64
from datetime import datetime
from planetstack.config import Config
from util.logger import Logger, logging
from observer.steps import *
from django.db.models import F, Q

logger = Logger(level=logging.INFO)

class FailedDependency(Exception):
    pass

class SyncStep:
    """ A PlanetStack Sync step.

    Attributes:
        psmodel        Model name the step synchronizes
        dependencies    list of names of models that must be synchronized first if the current model depends on them
    """
    slow=False
    def get_prop(prop):
        try:
            sync_config_dir = Config().sync_config_dir
        except:
            sync_config_dir = '/etc/planetstack/sync'
        prop_config_path = '/'.join(sync_config_dir,self.name,prop)
        return open(prop_config_path).read().rstrip()

    def __init__(self, **args):
        """Initialize a sync step
           Keyword arguments:
                   name -- Name of the step
                provides -- PlanetStack models sync'd by this step
        """
        dependencies = []
        self.driver = args.get('driver')
        self.error_map = args.get('error_map')

        try:
            self.soft_deadline = int(self.get_prop('soft_deadline_seconds'))
        except:
            self.soft_deadline = 5 # 5 seconds

        return

    def fetch_pending(self, deletion=False):
        # This is the most common implementation of fetch_pending
        # Steps should override it if they have their own logic
        # for figuring out what objects are outstanding.
        main_obj = self.provides[0]
        if (not deletion):
            objs = main_obj.objects.filter(Q(enacted__lt=F('updated')) | Q(enacted=None))
        else:
            objs = main_obj.deleted_objects.all()

        return objs
        #return Sliver.objects.filter(ip=None)

    def check_dependencies(self, obj, failed):
        for dep in self.dependencies:
            peer_name = dep[0].lower() + dep[1:]    # django names are camelCased with the first letter lower
            try:
                peer_object = getattr(obj, peer_name)
            except:
                peer_object = None

            if (peer_object and peer_object.pk==failed.pk and type(peer_object)==type(failed)):
                if (obj.backend_status!=peer_object.backend_status):
                    obj.backend_status = peer_object.backend_status
                    obj.save(update_fields=['backend_status'])
                raise FailedDependency("Failed dependency for %s:%s peer %s:%s failed  %s:%s" % (obj.__class__.__name__, str(obj.pk), peer_object.__class__.__name__, str(peer_object.pk), failed.__class__.__name__, str(failed.pk)))

    def call(self, failed=[], deletion=False):
        pending = self.fetch_pending(deletion)
        for o in pending:
            try:
                for f in failed:
                    self.check_dependencies(o,f) # Raises exception if failed
                if (deletion):
                    self.delete_record(o)
                    o.delete(purge=True)
                else:
                    self.sync_record(o)
                    o.enacted = datetime.now() # Is this the same timezone? XXX
                    o.backend_status = "OK"
                    o.save(update_fields=['enacted'])
            except Exception,e:
                try:
                    o.backend_status = self.error_map.map(str(e))
                except:
                    o.backend_status = str(e)

                if (o.pk):
                    o.save(update_fields=['backend_status'])

                logger.log_exc("sync step failed!")
                failed.append(o)

        return failed

    def __call__(self, **args):
        return self.call(**args)
<|endoftext|>


<commit_before>import logging
import xml.sax.handler
import os
from popen2 import Popen3
from select import select

from campus_factory.util.ExternalCommands import RunExternal

class AvailableGlideins(xml.sax.handler.ContentHandler, object):

    # Command to query the collector for available glideins
    command = "condor_status -avail -const '(IsUndefined(Offline) == True) || (Offline == false)' -format '<glidein name=\"%s\"/>' 'Name'"

    def __init__(self):
	self.owner_idle = {}
        pass

    def GetIdle(self):
        self.idle = 0
        self.found = False

        # Get the xml from the collector
        to_parse, stderr = RunExternal(self.command)
        formatted_to_parse = "<doc>%s</doc>" % to_parse

        # Parse the data
        try:
            xml.sax.parseString(formatted_to_parse, self)
        except xml.sax._exceptions.SAXParseException, inst:
            logging.error("Error parsing:")
            logging.error("command = %s" % self.command)
            logging.error("stderr = %s" % stderr)
            logging.error("stdout = %s" % to_parse)
            logging.error("Error: %s - %s" % ( str(inst), inst.args ))

        if not self.found and (len(stderr) != 0):
            logging.error("No valid output received from command: %s"% self.command)
            logging.error("stderr = %s" % stderr)
            logging.error("stdout = %s" % to_parse)
            return None

        return self.idle

    def Run(self):
        """
        Generic function for when this class is inherited
        """
        return self.GetIdle()

    def startElement(self, name, attributes):
        if name == "glidein":
            self.idle += 1
            self.found = True

            if not self.owner_idle.has_key(attributes['owner']):
                self.owner_idle[attributes['owner']] = 0
            self.owner_idle[attributes['owner']] += 1

    def GetOwnerIdle(self):
        return self.owner_idle

class IdleGlideins(AvailableGlideins):

    command = "condor_q -const '(GlideinJob == true) && (JobStatus == 1)' -format '<glidein owner=\"%s\"/>' 'Owner'"

class IdleJobs(AvailableGlideins):

    command = "condor_q -name %s -const '(GlideinJob =!= true) &&  (JobStatus == 1) && (JobUniverse == 5)' -format '<glidein owner=\"%s\"/>' 'Owner'"

    def __init__(self, schedd):
        super(IdleJobs, self).__init__()
        self.command = self.command % schedd

class IdleLocalJobs(AvailableGlideins):

    command = "condor_q -const '(GlideinJob =!= true) &&  (JobStatus == 1) && (JobUniverse == 5)' -format '<glidein owner=\"%s\"/>' 'Owner'"

class FactoryID(AvailableGlideins):
    command = "condor_q -const '(IsUndefined(IsFactory) == FALSE)' -format '<factory id=\"%s\"/>' 'ClusterId'"

    def startElement(self, name, attributes):
        if name == "factory":
            self.factory_id = attributes.getValue("id")
            self.found = True

    def GetId(self):
        self.GetIdle()
        return self.factory_id

class RunningGlideinsJobs(AvailableGlideins):
    """
    Gets the number of running glidein jobs (condor_q)
    """

    command = "condor_q -const '(GlideinJob == true) && (JobStatus == 2)' -format '<glidein owner=\"%s\"/>' 'Owner'"

class RunningGlideins(AvailableGlideins):
    """
    Returns the number of startd's reporting to the collector (condor_status)
    """

    command = "condor_status -const '(IsUndefined(IS_GLIDEIN) == FALSE) && (IS_GLIDEIN == TRUE) && (IsUndefined(Offline))' -format '<glidein name=\"%s\"/>' 'Name'"

<commit_msg>Campus Factory: Fixing percent sign for multiple clusters<commit_after>import logging
import xml.sax.handler
import os
from popen2 import Popen3
from select import select

from campus_factory.util.ExternalCommands import RunExternal

class AvailableGlideins(xml.sax.handler.ContentHandler, object):

    # Command to query the collector for available glideins
    command = "condor_status -avail -const '(IsUndefined(Offline) == True) || (Offline == false)' -format '<glidein name=\"%s\"/>' 'Name'"

    def __init__(self):
	self.owner_idle = {}
        pass

    def GetIdle(self):
        self.idle = 0
        self.found = False

        # Get the xml from the collector
        to_parse, stderr = RunExternal(self.command)
        formatted_to_parse = "<doc>%s</doc>" % to_parse

        # Parse the data
        try:
            xml.sax.parseString(formatted_to_parse, self)
        except xml.sax._exceptions.SAXParseException, inst:
            logging.error("Error parsing:")
            logging.error("command = %s" % self.command)
            logging.error("stderr = %s" % stderr)
            logging.error("stdout = %s" % to_parse)
            logging.error("Error: %s - %s" % ( str(inst), inst.args ))

        if not self.found and (len(stderr) != 0):
            logging.error("No valid output received from command: %s"% self.command)
            logging.error("stderr = %s" % stderr)
            logging.error("stdout = %s" % to_parse)
            return None

        return self.idle

    def Run(self):
        """
        Generic function for when this class is inherited
        """
        return self.GetIdle()

    def startElement(self, name, attributes):
        if name == "glidein":
            self.idle += 1
            self.found = True

            if not self.owner_idle.has_key(attributes['owner']):
                self.owner_idle[attributes['owner']] = 0
            self.owner_idle[attributes['owner']] += 1

    def GetOwnerIdle(self):
        return self.owner_idle

class IdleGlideins(AvailableGlideins):

    command = "condor_q -const '(GlideinJob == true) && (JobStatus == 1)' -format '<glidein owner=\"%s\"/>' 'Owner'"

class IdleJobs(AvailableGlideins):

    command = "condor_q -name %s -const '(GlideinJob =!= true) &&  (JobStatus == 1) && (JobUniverse == 5)' -format '<glidein owner=\"%%s\"/>' 'Owner'"

    def __init__(self, schedd):
        super(IdleJobs, self).__init__()
        self.command = self.command % schedd

class IdleLocalJobs(AvailableGlideins):

    command = "condor_q -const '(GlideinJob =!= true) &&  (JobStatus == 1) && (JobUniverse == 5)' -format '<glidein owner=\"%s\"/>' 'Owner'"

class FactoryID(AvailableGlideins):
    command = "condor_q -const '(IsUndefined(IsFactory) == FALSE)' -format '<factory id=\"%s\"/>' 'ClusterId'"

    def startElement(self, name, attributes):
        if name == "factory":
            self.factory_id = attributes.getValue("id")
            self.found = True

    def GetId(self):
        self.GetIdle()
        return self.factory_id

class RunningGlideinsJobs(AvailableGlideins):
    """
    Gets the number of running glidein jobs (condor_q)
    """

    command = "condor_q -const '(GlideinJob == true) && (JobStatus == 2)' -format '<glidein owner=\"%s\"/>' 'Owner'"

class RunningGlideins(AvailableGlideins):
    """
    Returns the number of startd's reporting to the collector (condor_status)
    """

    command = "condor_status -const '(IsUndefined(IS_GLIDEIN) == FALSE) && (IS_GLIDEIN == TRUE) && (IsUndefined(Offline))' -format '<glidein name=\"%s\"/>' 'Name'"

<|endoftext|>


<commit_before>from __future__ import absolute_import, division, print_function

import sys
from operator import itemgetter
import operator
from toolz import compose, identity
from collections import Iterator

from blaze.expr.table import *
from blaze.expr.table import count as Count
from . import core, python
from .python import compute, rowfunc, RowWise, listpack
from ..compatibility import builtins
from ..expr import table
from ..dispatch import dispatch

from toolz.curried import get

__all__ = ['compute', 'into', 'RDD', 'pyspark']

try:
    from itertools import compress, chain
    import pyspark
    from pyspark.rdd import RDD
except ImportError:
    #Create a dummy RDD for py 2.6
    class Dummy(object):
        sum = max = min = count = distinct = mean = variance = stdev = None
    pyspark = Dummy()
    pyspark.rdd = Dummy()
    RDD = Dummy

# PySpark adds a SIGCHLD signal handler, but that breaks other packages, so we
# remove it
# See https://issues.apache.org/jira/browse/SPARK-1394
try:
    import signal
    signal.signal(signal.SIGCHLD, signal.SIG_DFL)
except:
    pass

@dispatch(RowWise, RDD)
def compute(t, rdd):
    rdd = compute(t.parent, rdd)
    func = rowfunc(t)
    return rdd.map(func)

@dispatch(TableSymbol, RDD)
def compute(t, s):
    return s

@dispatch(Selection, RDD)
def compute(t, rdd):
    rdd = compute(t.parent, rdd)
    predicate = rowfunc(t.predicate)
    return rdd.filter(predicate)

rdd_reductions = {
        table.sum: RDD.sum,
        table.min: RDD.min,
        table.max: RDD.max,
        table.count: RDD.count,
        table.mean: RDD.mean,
        table.var: RDD.variance,
        table.std: RDD.stdev,
        table.nunique: compose(RDD.count, RDD.distinct)}

@dispatch(tuple(rdd_reductions), RDD)
def compute(t, rdd):
    reduction = rdd_reductions[type(t)]
    return reduction(compute(t.parent, rdd))

def istruthy(x):
    return not not x

@dispatch(table.any, RDD)
def compute(t, rdd):
    rdd = compute(t.parent, rdd)
    return istruthy(rdd.filter(identity).take(1))

@dispatch(table.all, RDD)
def compute(t, rdd):
    rdd = compute(t.parent, rdd)
    return not rdd.filter(lambda x: not x).take(1)

@dispatch(Head, RDD)
def compute(t, rdd):
    rdd = compute(t.parent, rdd)
    return rdd.take(t.n)

@dispatch(Sort, RDD)
def compute(t, rdd):
    rdd = compute(t.parent, rdd)
    func = rowfunc(t[t.column])
    return (rdd.keyBy(func)
                .sortByKey(ascending=t.ascending)
                .map(lambda x: x[1]))

@dispatch(Distinct, RDD)
def compute(t, rdd):
    rdd = compute(t.parent, rdd)
    return rdd.distinct()

@dispatch(Join, RDD, RDD)
def compute(t, lhs, rhs):
    lhs = compute(t.lhs, lhs)
    rhs = compute(t.rhs, rhs)

    on_left = rowfunc(t.lhs[t.on_left])
    on_right = rowfunc(t.rhs[t.on_right])

    lhs = lhs.keyBy(on_left)
    rhs = rhs.keyBy(on_right)

    # Calculate the indices we want in the joined table
    columns = t.lhs.columns + t.rhs.columns
    repeated_index = set([len(t.lhs.columns) + t.rhs.columns.index(col)
            for col in listpack(t.on_right)])

    wanted = [i for i in range(len(columns)) if i not in repeated_index]

    getter = get(wanted)
    reassemble = lambda x: getter(x[1][0] + x[1][1])

    return lhs.join(rhs).map(reassemble)

python_reductions = {
              table.sum: builtins.sum,
              table.count: builtins.len,
              table.max: builtins.max,
              table.min: builtins.min,
              table.any: builtins.any,
              table.all: builtins.all,
              table.mean: python._mean,
              table.var: python._var,
              table.std: python._std,
              table.nunique: lambda x: len(set(x))}

@dispatch(By, RDD)
def compute(t, rdd):
    rdd = compute(t.parent, rdd)
    try:
        reduction = python_reductions[type(t.apply)]
    except KeyError:
        raise NotImplementedError("By only implemented for common reductions."
                                  "\nGot %s" % type(t.apply))

    grouper = rowfunc(t.grouper)
    pre = rowfunc(t.apply.parent)

    groups = (rdd.map(lambda x: (grouper(x), pre(x)))
             .groupByKey())

    if isinstance(t.grouper, (Column, ColumnWise)):
        func = lambda x: (x[0], reduction(x[1]))
    else:
        func = lambda x: (tuple(x[0]) + (reduction(x[1]),))
    return groups.map(func)

@dispatch((Label, ReLabel), RDD)
def compute(t, rdd):
    return compute(t.parent, rdd)

@dispatch(RDD, RDD)
def into(a, b):
    return b

@dispatch(pyspark.SparkContext, (list, tuple, Iterator))
def into(sc, seq):
    return sc.parallelize(seq)

@dispatch(RDD, (list, tuple))
def into(rdd, seq):
    return into(rdd.context, seq)

@dispatch(list, RDD)
def into(seq, rdd):
    return rdd.collect()
<commit_msg>avoid spark issue for into if spark is not implemented<commit_after>from __future__ import absolute_import, division, print_function

import sys
from operator import itemgetter
import operator
from toolz import compose, identity
from collections import Iterator

from blaze.expr.table import *
from blaze.expr.table import count as Count
from . import core, python
from .python import compute, rowfunc, RowWise, listpack
from ..compatibility import builtins
from ..expr import table
from ..dispatch import dispatch

from toolz.curried import get

__all__ = ['compute', 'into', 'RDD', 'pyspark', 'SparkContext']

try:
    from itertools import compress, chain
    from pyspark import SparkContext
    import pyspark
    from pyspark.rdd import RDD
except ImportError:
    #Create a dummy RDD for py 2.6
    class Dummy(object):
        sum = max = min = count = distinct = mean = variance = stdev = None
    SparkContext = Dummy
    pyspark = Dummy()
    pyspark.rdd = Dummy()
    RDD = Dummy

# PySpark adds a SIGCHLD signal handler, but that breaks other packages, so we
# remove it
# See https://issues.apache.org/jira/browse/SPARK-1394
try:
    import signal
    signal.signal(signal.SIGCHLD, signal.SIG_DFL)
except:
    pass

@dispatch(RowWise, RDD)
def compute(t, rdd):
    rdd = compute(t.parent, rdd)
    func = rowfunc(t)
    return rdd.map(func)

@dispatch(TableSymbol, RDD)
def compute(t, s):
    return s

@dispatch(Selection, RDD)
def compute(t, rdd):
    rdd = compute(t.parent, rdd)
    predicate = rowfunc(t.predicate)
    return rdd.filter(predicate)

rdd_reductions = {
        table.sum: RDD.sum,
        table.min: RDD.min,
        table.max: RDD.max,
        table.count: RDD.count,
        table.mean: RDD.mean,
        table.var: RDD.variance,
        table.std: RDD.stdev,
        table.nunique: compose(RDD.count, RDD.distinct)}

@dispatch(tuple(rdd_reductions), RDD)
def compute(t, rdd):
    reduction = rdd_reductions[type(t)]
    return reduction(compute(t.parent, rdd))

def istruthy(x):
    return not not x

@dispatch(table.any, RDD)
def compute(t, rdd):
    rdd = compute(t.parent, rdd)
    return istruthy(rdd.filter(identity).take(1))

@dispatch(table.all, RDD)
def compute(t, rdd):
    rdd = compute(t.parent, rdd)
    return not rdd.filter(lambda x: not x).take(1)

@dispatch(Head, RDD)
def compute(t, rdd):
    rdd = compute(t.parent, rdd)
    return rdd.take(t.n)

@dispatch(Sort, RDD)
def compute(t, rdd):
    rdd = compute(t.parent, rdd)
    func = rowfunc(t[t.column])
    return (rdd.keyBy(func)
                .sortByKey(ascending=t.ascending)
                .map(lambda x: x[1]))

@dispatch(Distinct, RDD)
def compute(t, rdd):
    rdd = compute(t.parent, rdd)
    return rdd.distinct()

@dispatch(Join, RDD, RDD)
def compute(t, lhs, rhs):
    lhs = compute(t.lhs, lhs)
    rhs = compute(t.rhs, rhs)

    on_left = rowfunc(t.lhs[t.on_left])
    on_right = rowfunc(t.rhs[t.on_right])

    lhs = lhs.keyBy(on_left)
    rhs = rhs.keyBy(on_right)

    # Calculate the indices we want in the joined table
    columns = t.lhs.columns + t.rhs.columns
    repeated_index = set([len(t.lhs.columns) + t.rhs.columns.index(col)
            for col in listpack(t.on_right)])

    wanted = [i for i in range(len(columns)) if i not in repeated_index]

    getter = get(wanted)
    reassemble = lambda x: getter(x[1][0] + x[1][1])

    return lhs.join(rhs).map(reassemble)

python_reductions = {
              table.sum: builtins.sum,
              table.count: builtins.len,
              table.max: builtins.max,
              table.min: builtins.min,
              table.any: builtins.any,
              table.all: builtins.all,
              table.mean: python._mean,
              table.var: python._var,
              table.std: python._std,
              table.nunique: lambda x: len(set(x))}

@dispatch(By, RDD)
def compute(t, rdd):
    rdd = compute(t.parent, rdd)
    try:
        reduction = python_reductions[type(t.apply)]
    except KeyError:
        raise NotImplementedError("By only implemented for common reductions."
                                  "\nGot %s" % type(t.apply))

    grouper = rowfunc(t.grouper)
    pre = rowfunc(t.apply.parent)

    groups = (rdd.map(lambda x: (grouper(x), pre(x)))
             .groupByKey())

    if isinstance(t.grouper, (Column, ColumnWise)):
        func = lambda x: (x[0], reduction(x[1]))
    else:
        func = lambda x: (tuple(x[0]) + (reduction(x[1]),))
    return groups.map(func)

@dispatch((Label, ReLabel), RDD)
def compute(t, rdd):
    return compute(t.parent, rdd)

@dispatch(RDD, RDD)
def into(a, b):
    return b

@dispatch(SparkContext, (list, tuple, Iterator))
def into(sc, seq):
    return sc.parallelize(seq)

@dispatch(RDD, (list, tuple))
def into(rdd, seq):
    return into(rdd.context, seq)

@dispatch(list, RDD)
def into(seq, rdd):
    return rdd.collect()
<|endoftext|>


<commit_before>from __future__ import absolute_import, division, print_function

from .expressions import Expr, ElemWise
from datashape import dshape, Record, DataShape, Unit, Option, date_, datetime_
import datashape

__all__ = ['DateTime', 'Date', 'date', 'Year', 'year', 'Month', 'month', 'Day',
        'day', 'Hour', 'hour', 'Second', 'second', 'Millisecond',
        'millisecond', 'Microsecond', 'microsecond', 'Date', 'date', 'Time',
        'time', 'UTCFromTimestamp']

class DateTime(ElemWise):
    """ Superclass for datetime accessors """
    __slots__ = '_child',

    def __str__(self):
        return '%s.%s' % (str(self._child), type(self).__name__.lower())

    @property
    def schema(self):
        return dshape(self._dtype)

    @property
    def _name(self):
        return '%s_%s' % (self._child._name, self.attr)

    @property
    def attr(self):
        return type(self).__name__.lower()

class Date(DateTime):
    _dtype = datashape.date_

def date(expr):
    return Date(expr)

class Year(DateTime):
    _dtype = datashape.int32

def year(expr):
    return Year(expr)

class Month(DateTime):
    _dtype = datashape.int32

def month(expr):
    return Month(expr)

class Day(DateTime):
    _dtype = datashape.int32

def day(expr):
    return Day(expr)

class Time(DateTime):
    _dtype = datashape.time_

def time(expr):
    return Time(Expr)

class Hour(DateTime):
    _dtype = datashape.int32

def hour(expr):
    return Hour(expr)

class Minute(DateTime):
    _dtype = datashape.int32

def minute(expr):
    return Minute(expr)

class Second(DateTime):
    _dtype = datashape.int32

def second(expr):
    return Second(expr)

class Millisecond(DateTime):
    _dtype = datashape.int64

def millisecond(expr):
    return Millisecond(expr)

class Microsecond(DateTime):
    _dtype = datashape.int64

def microsecond(expr):
    return Microsecond(expr)

class UTCFromTimestamp(DateTime):
    _dtype = datashape.datetime_

def utcfromtimestamp(expr):
    return UTCFromTimestamp(expr)

precisions = ['year', 'month', 'week', 'day', 'hour', 'minute', 'second',
'millisecond', 'microsecond', 'nanosecond']

_precision_aliases = {'y': 'year', 'w': 'week', 'd': 'day', 'date': 'day',
    'h': 'hour', 's': 'second', 'ms': 'millisecond', 'us': 'microsecond',
    'ns': 'nanosecond'}

def normalize_time_unit(s):
    """ Normalize time input to one of 'year', 'second', 'millisecond', etc..

    Example
    -------

    >>> normalize_time_unit('milliseconds')
    'millisecond'
    >>> normalize_time_unit('ms')
    'millisecond'
    """
    s = s.lower().strip()
    if s in precisions:
        return s
    if s in _precision_aliases:
        return _precision_aliases[s]
    if s[-1] == 's':
        return normalize_time_unit(s.rstrip('s'))

    raise ValueError("Do not understand time unit %s" % s)

from .expressions import schema_method_list, method_properties
from datashape.predicates import isdatelike, isnumeric

schema_method_list.extend([
    (isdatelike, set([year, month, day, hour, minute, date, time, second,
                      millisecond, microsecond])),
    (isnumeric, set([utcfromtimestamp]))
    ])

method_properties |= set([year, month, day, hour, minute, second, millisecond,
                          microsecond, date, time, utcfromtimestamp])
<commit_msg>expr: datetime truncation operation<commit_after>from __future__ import absolute_import, division, print_function

from .expressions import Expr, ElemWise
from datashape import dshape, Record, DataShape, Unit, Option, date_, datetime_
import datashape

__all__ = ['DateTime', 'Date', 'date', 'Year', 'year', 'Month', 'month', 'Day',
        'day', 'Hour', 'hour', 'Second', 'second', 'Millisecond',
        'millisecond', 'Microsecond', 'microsecond', 'Date', 'date', 'Time',
        'time', 'UTCFromTimestamp', 'DateTimeTruncate']

class DateTime(ElemWise):
    """ Superclass for datetime accessors """
    __slots__ = '_child',

    def __str__(self):
        return '%s.%s' % (str(self._child), type(self).__name__.lower())

    @property
    def schema(self):
        return dshape(self._dtype)

    @property
    def _name(self):
        return '%s_%s' % (self._child._name, self.attr)

    @property
    def attr(self):
        return type(self).__name__.lower()

class Date(DateTime):
    _dtype = datashape.date_

def date(expr):
    return Date(expr)

class Year(DateTime):
    _dtype = datashape.int32

def year(expr):
    return Year(expr)

class Month(DateTime):
    _dtype = datashape.int32

def month(expr):
    return Month(expr)

class Day(DateTime):
    _dtype = datashape.int32

def day(expr):
    return Day(expr)

class Time(DateTime):
    _dtype = datashape.time_

def time(expr):
    return Time(Expr)

class Hour(DateTime):
    _dtype = datashape.int32

def hour(expr):
    return Hour(expr)

class Minute(DateTime):
    _dtype = datashape.int32

def minute(expr):
    return Minute(expr)

class Second(DateTime):
    _dtype = datashape.int32

def second(expr):
    return Second(expr)

class Millisecond(DateTime):
    _dtype = datashape.int64

def millisecond(expr):
    return Millisecond(expr)

class Microsecond(DateTime):
    _dtype = datashape.int64

def microsecond(expr):
    return Microsecond(expr)

class UTCFromTimestamp(DateTime):
    _dtype = datashape.datetime_

def utcfromtimestamp(expr):
    return UTCFromTimestamp(expr)

units = ['year', 'month', 'week', 'day', 'hour', 'minute', 'second',
'millisecond', 'microsecond', 'nanosecond']

_unit_aliases = {'y': 'year', 'w': 'week', 'd': 'day', 'date': 'day',
    'h': 'hour', 's': 'second', 'ms': 'millisecond', 'us': 'microsecond',
    'ns': 'nanosecond'}

def normalize_time_unit(s):
    """ Normalize time input to one of 'year', 'second', 'millisecond', etc..

    Example
    -------

    >>> normalize_time_unit('milliseconds')
    'millisecond'
    >>> normalize_time_unit('ms')
    'millisecond'
    """
    s = s.lower().strip()
    if s in units:
        return s
    if s in _unit_aliases:
        return _unit_aliases[s]
    if s[-1] == 's':
        return normalize_time_unit(s.rstrip('s'))

    raise ValueError("Do not understand time unit %s" % s)

class DateTimeTruncate(DateTime):
    __slots__ = '_child', 'measure', 'unit'

    @property
    def _dtype(self):
        if units.index('day') >= units.index(self.unit):
            return datashape.date_
        else:
            return datashape.datetime_

def truncate(expr, measure, unit):
    """ Truncate datetime expression

    Example
    -------

    >>> s = Symbol('s', 'datetime')
    >>> compute(s.truncate(10, 'minute'),
    ...         datetime(2000, 06, 25, 12, 35, 10)
    datetime.datetime(2000, 06, 25, 12, 30, 00)

    >>> compute(s.truncate(1, 'week'),
    ...         datetime(2000, 06, 25, 12, 35, 10)
    datetime.date(2000, 06, 22)
    """
    return DateTimeTruncate(expr, measure, normalize_time_unit(unit))

from .expressions import schema_method_list, method_properties
from datashape.predicates import isdatelike, isnumeric

schema_method_list.extend([
    (isdatelike, set([year, month, day, hour, minute, date, time, second,
                      millisecond, microsecond, truncate])),
    (isnumeric, set([utcfromtimestamp]))
    ])

method_properties |= set([year, month, day, hour, minute, second, millisecond,
                          microsecond, date, time, utcfromtimestamp])
<|endoftext|>


<commit_before>"""Defines a sync module for Blink."""

import logging

from requests.structures import CaseInsensitiveDict
from blinkpy import api
from blinkpy.camera import BlinkCamera
from blinkpy.helpers.constants import ONLINE

_LOGGER = logging.getLogger(__name__)

class BlinkSyncModule():
    """Class to initialize sync module."""

    def __init__(self, blink, network_name, network_id):
        """
        Initialize Blink sync module.

        :param blink: Blink class instantiation
        """
        self.blink = blink
        self._auth_header = blink.auth_header
        self.network_id = network_id
        self.region = blink.region
        self.region_id = blink.region_id
        self.name = network_name
        self.serial = None
        self.status = None
        self.sync_id = None
        self.host = None
        self.summary = None
        self.homescreen = None
        self.network_info = None
        self.record_dates = {}
        self.videos = {}
        self.events = []
        self.cameras = CaseInsensitiveDict({})
        self.all_clips = {}

    @property
    def attributes(self):
        """Return sync attributes."""
        attr = {
            'name': self.name,
            'id': self.sync_id,
            'network_id': self.network_id,
            'serial': self.serial,
            'status': self.status,
            'region': self.region,
            'region_id': self.region_id,
        }
        return attr

    @property
    def urls(self):
        """Return device urls."""
        return self.blink.urls

    @property
    def online(self):
        """Return boolean system online status."""
        return ONLINE[self.status]

    @property
    def arm(self):
        """Return status of sync module: armed/disarmed."""
        return self.network_info['network']['armed']

    @arm.setter
    def arm(self, value):
        """Arm or disarm system."""
        if value:
            return api.request_system_arm(self.blink, self.network_id)

        return api.request_system_disarm(self.blink, self.network_id)

    def start(self):
        """Initialize the system."""
        response = api.request_syncmodule(self.blink, self.network_id)
        self.summary = response['syncmodule']
        self.sync_id = self.summary['id']
        self.network_id = self.summary['network_id']
        self.serial = self.summary['serial']
        self.status = self.summary['status']

        self.events = self.get_events()

        self.homescreen = api.request_homescreen(self.blink)

        self.network_info = api.request_network_status(self.blink,
                                                       self.network_id)

        camera_info = self.get_camera_info()
        for camera_config in camera_info:
            name = camera_config['name']
            self.cameras[name] = BlinkCamera(self)

        self.videos = self.get_videos()
        for camera_config in camera_info:
            name = camera_config['name']
            if name in self.cameras:
                self.cameras[name].update(camera_config, force_cache=True)

    def get_events(self):
        """Retrieve events from server."""
        response = api.request_sync_events(self.blink, self.network_id)
        return response['event']

    def get_camera_info(self):
        """Retrieve camera information."""
        response = api.request_cameras(self.blink, self.network_id)
        return response['devicestatus']

    def refresh(self, force_cache=False):
        """Get all blink cameras and pulls their most recent status."""
        self.events = self.get_events()
        self.videos = self.get_videos()
        self.homescreen = api.request_homescreen(self.blink)
        self.network_info = api.request_network_status(self.blink,
                                                       self.network_id)
        camera_info = self.get_camera_info()
        for camera_config in camera_info:
            name = camera_config['name']
            self.cameras[name].update(camera_config, force_cache=force_cache)

    def get_videos(self, start_page=0, end_page=1):
        """
        Retrieve last recorded videos per camera.

        :param start_page: Page to start reading from on blink servers
                           (defaults to 0)
        :param end_page: Page to stop reading from (defaults to 1)
        """
        videos = list()
        all_dates = dict()

        for page_num in range(start_page, end_page + 1):
            this_page = api.request_videos(self.blink, page=page_num)
            if not this_page:
                break
            videos.append(this_page)
        _LOGGER.debug("Getting videos from page %s through %s",
                      start_page,
                      end_page)
        for page in videos:
            for entry in page:
                camera_name = entry['camera_name']
                clip_addr = entry['address']
                thumb_addr = entry['thumbnail']
                clip_date = clip_addr.split('_')[-6:]
                clip_date = '_'.join(clip_date)
                clip_date = clip_date.split('.')[0]
                try:
                    self.all_clips[camera_name][clip_date] = clip_addr
                except KeyError:
                    self.all_clips[camera_name] = {clip_date: clip_addr}

                if camera_name not in all_dates:
                    all_dates[camera_name] = list()
                all_dates[camera_name].append(clip_date)
                try:
                    self.videos[camera_name].append(
                        {
                            'clip': clip_addr,
                            'thumb': thumb_addr,
                        }
                    )
                except KeyError:
                    self.videos[camera_name] = [
                        {
                            'clip': clip_addr,
                            'thumb': thumb_addr,
                        }
                    ]
        self.record_dates = all_dates
        _LOGGER.debug("Retrieved a total of %s records", len(all_dates))
        return self.videos
<commit_msg>Added some error handling for video method due to api changes<commit_after>"""Defines a sync module for Blink."""

import logging

from requests.structures import CaseInsensitiveDict
from blinkpy import api
from blinkpy.camera import BlinkCamera
from blinkpy.helpers.constants import ONLINE

_LOGGER = logging.getLogger(__name__)

class BlinkSyncModule():
    """Class to initialize sync module."""

    def __init__(self, blink, network_name, network_id):
        """
        Initialize Blink sync module.

        :param blink: Blink class instantiation
        """
        self.blink = blink
        self._auth_header = blink.auth_header
        self.network_id = network_id
        self.region = blink.region
        self.region_id = blink.region_id
        self.name = network_name
        self.serial = None
        self.status = None
        self.sync_id = None
        self.host = None
        self.summary = None
        self.homescreen = None
        self.network_info = None
        self.record_dates = {}
        self.videos = {}
        self.events = []
        self.cameras = CaseInsensitiveDict({})
        self.all_clips = {}

    @property
    def attributes(self):
        """Return sync attributes."""
        attr = {
            'name': self.name,
            'id': self.sync_id,
            'network_id': self.network_id,
            'serial': self.serial,
            'status': self.status,
            'region': self.region,
            'region_id': self.region_id,
        }
        return attr

    @property
    def urls(self):
        """Return device urls."""
        return self.blink.urls

    @property
    def online(self):
        """Return boolean system online status."""
        return ONLINE[self.status]

    @property
    def arm(self):
        """Return status of sync module: armed/disarmed."""
        return self.network_info['network']['armed']

    @arm.setter
    def arm(self, value):
        """Arm or disarm system."""
        if value:
            return api.request_system_arm(self.blink, self.network_id)

        return api.request_system_disarm(self.blink, self.network_id)

    def start(self):
        """Initialize the system."""
        response = api.request_syncmodule(self.blink, self.network_id)
        self.summary = response['syncmodule']
        self.sync_id = self.summary['id']
        self.network_id = self.summary['network_id']
        self.serial = self.summary['serial']
        self.status = self.summary['status']

        self.events = self.get_events()

        self.homescreen = api.request_homescreen(self.blink)

        self.network_info = api.request_network_status(self.blink,
                                                       self.network_id)

        camera_info = self.get_camera_info()
        for camera_config in camera_info:
            name = camera_config['name']
            self.cameras[name] = BlinkCamera(self)

        self.videos = self.get_videos()
        for camera_config in camera_info:
            name = camera_config['name']
            if name in self.cameras:
                self.cameras[name].update(camera_config, force_cache=True)

    def get_events(self):
        """Retrieve events from server."""
        response = api.request_sync_events(self.blink, self.network_id)
        return response['event']

    def get_camera_info(self):
        """Retrieve camera information."""
        response = api.request_cameras(self.blink, self.network_id)
        return response['devicestatus']

    def refresh(self, force_cache=False):
        """Get all blink cameras and pulls their most recent status."""
        self.events = self.get_events()
        self.videos = self.get_videos()
        self.homescreen = api.request_homescreen(self.blink)
        self.network_info = api.request_network_status(self.blink,
                                                       self.network_id)
        camera_info = self.get_camera_info()
        for camera_config in camera_info:
            name = camera_config['name']
            self.cameras[name].update(camera_config, force_cache=force_cache)

    def get_videos(self, start_page=0, end_page=1):
        """
        Retrieve last recorded videos per camera.

        :param start_page: Page to start reading from on blink servers
                           (defaults to 0)
        :param end_page: Page to stop reading from (defaults to 1)
        """
        videos = list()
        all_dates = dict()

        for page_num in range(start_page, end_page + 1):
            this_page = api.request_videos(self.blink, page=page_num)
            if not this_page:
                break
            elif 'message' in this_page:
                _LOGGER.warning("Could not retrieve videos. Message: %s",
                                this_page['message'])
                break

            videos.append(this_page)
        _LOGGER.debug("Getting videos from page %s through %s",
                      start_page,
                      end_page)
        for page in videos:
            for entry in page:
                try:
                    camera_name = entry['camera_name']
                    clip_addr = entry['address']
                    thumb_addr = entry['thumbnail']
                except TypeError:
                    _LOGGER.warning("Could not extract video information.")
                    break
                clip_date = clip_addr.split('_')[-6:]
                clip_date = '_'.join(clip_date)
                clip_date = clip_date.split('.')[0]
                try:
                    self.all_clips[camera_name][clip_date] = clip_addr
                except KeyError:
                    self.all_clips[camera_name] = {clip_date: clip_addr}

                if camera_name not in all_dates:
                    all_dates[camera_name] = list()
                all_dates[camera_name].append(clip_date)
                try:
                    self.videos[camera_name].append(
                        {
                            'clip': clip_addr,
                            'thumb': thumb_addr,
                        }
                    )
                except KeyError:
                    self.videos[camera_name] = [
                        {
                            'clip': clip_addr,
                            'thumb': thumb_addr,
                        }
                    ]
        self.record_dates = all_dates
        _LOGGER.debug("Retrieved a total of %s records", len(all_dates))
        return self.videos
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
#
# This file is part of Invenio.
# Copyright (C) 2016-2022 CERN.
#
# Invenio is free software; you can redistribute it and/or modify it
# under the terms of the MIT License; see LICENSE file for more details.

"""Invenio digital library framework."""

from .ext import InvenioCommunities
from .proxies import current_communities

__version__ = "2.8.0.dev24"

__all__ = (
    'InvenioCommunities',
    'current_communities'
)
<commit_msg>release: v2.8.0.dev25<commit_after># -*- coding: utf-8 -*-
#
# This file is part of Invenio.
# Copyright (C) 2016-2022 CERN.
#
# Invenio is free software; you can redistribute it and/or modify it
# under the terms of the MIT License; see LICENSE file for more details.

"""Invenio digital library framework."""

from .ext import InvenioCommunities
from .proxies import current_communities

__version__ = "2.8.0.dev25"

__all__ = (
    'InvenioCommunities',
    'current_communities'
)
<|endoftext|>