<commit_before>import numpy
import timeit
import matplotlib.pyplot as plt
import sys
from settings import DATA_DIR, get_dir

numpy.random.seed(21)
num_repeats = 20
min_size = 500
max_size = 5001
step = 500

data_dir = get_dir(DATA_DIR)

def benchmark_numpy():
    times = []
    print("Benchmarking Numpy " + str(numpy.__version__))

    for i in range(min_size, max_size, step):
        print(i)
        global A, B
        A = numpy.random.rand(i, i).astype(numpy.float32)
        B = numpy.random.rand(i, i).astype(numpy.float32)

        current_times = [i]

        timer = timeit.Timer("numpy.dot(A, B)", "import numpy; from __main__ import A, B")
        current_times.append(numpy.min(timer.repeat(num_repeats, 1)))

        timer = timeit.Timer("numpy.exp(A)", "import numpy; from __main__ import A")
        current_times.append(numpy.min(timer.repeat(num_repeats, 1)))

        timer = timeit.Timer("numpy.sum(A)", "import numpy; from __main__ import A")
        current_times.append(numpy.min(timer.repeat(num_repeats, 1)))

        timer = timeit.Timer("A + B", "import numpy; from __main__ import A, B")
        current_times.append(numpy.min(timer.repeat(num_repeats, 1)))

        timer = timeit.Timer("numpy.mean(A)", "import numpy; from __main__ import A")
        current_times.append(numpy.min(timer.repeat(num_repeats, 1)))

        timer = timeit.Timer("numpy.min(A)", "import numpy; from __main__ import A")
        current_times.append(numpy.min(timer.repeat(num_repeats, 1)))

        times.append(current_times)

    times = numpy.array(times)
    print(times)
    numpy.save(data_dir + "numpy_times", times)

def benchmark_theano():
    from theano import function
    import theano
    import theano.tensor as T
    print("Benchmarking Theano " + str(theano.__version__))

    times = []

    for i in range(min_size, max_size, step):
        print(i)
        global A, B
        A = numpy.random.rand(i, i).astype(numpy.float32)
        B = numpy.random.rand(i, i).astype(numpy.float32)
        current_times = [i]

        global f
        A = theano.shared(A)
        B = theano.shared(B)

        z = T.dot(A, B)
        f = function([], z)
        timer = timeit.Timer("f()", "from __main__ import f")
        current_times.append(numpy.min(timer.repeat(num_repeats, 1)))

        z = T.exp(A)
        f = function([], z)
        timer = timeit.Timer("f()", "from __main__ import f")
        current_times.append(numpy.min(timer.repeat(num_repeats, 1)))

        z = T.sum(A)
        f = function([], z)
        timer = timeit.Timer("f()", "from __main__ import f")
        current_times.append(numpy.min(timer.repeat(num_repeats, 1)))

        z = T.add(A, B)
        f = function([], z)
        timer = timeit.Timer("f()", "from __main__ import f")
        current_times.append(numpy.min(timer.repeat(num_repeats, 1)))

        z = T.mean(A)
        f = function([], z)
        timer = timeit.Timer("f()", "from __main__ import f")
        current_times.append(numpy.min(timer.repeat(num_repeats, 1)))

        z = T.min(A)
        f = function([], z)
        timer = timeit.Timer("f()", "from __main__ import f")
        current_times.append(numpy.min(timer.repeat(num_repeats, 1)))

        times.append(current_times)

    times = numpy.array(times)
    print(times)
    numpy.save(data_dir + "theano_times", times)

def benchmark_tensorflow():
    import tensorflow
    print("Benchmarking Tensorflow " + str(tensorflow.__version__))

    times = []

    for i in range(min_size, max_size, step):
        print(i)
        global A, B
        A = numpy.random.rand(i, i).astype(numpy.float32)
        B = numpy.random.rand(i, i).astype(numpy.float32)
        current_times = [i]

        global f
        # Set up the tensorflow stuff
        A = tensorflow.constant(A)
        B = tensorflow.constant(B)
        global sess, result

        sess = tensorflow.Session()
        result = tensorflow.matmul(A, B)
        timer = timeit.Timer("sess.run(result)", setup="import tensorflow; from __main__ import sess, A, B, result")
        current_times.append(numpy.min(timer.repeat(num_repeats, 1)))
        sess.close()

        sess = tensorflow.Session()
        result = tensorflow.exp(A)
        timer = timeit.Timer("sess.run(result)", setup="import tensorflow; from __main__ import sess, A, B, result")
        current_times.append(numpy.min(timer.repeat(num_repeats, 1)))
        sess.close()

        sess = tensorflow.Session()
        result = tensorflow.reduce_sum(A)
        timer = timeit.Timer("sess.run(result)", setup="import tensorflow; from __main__ import sess, A, B, result")
        current_times.append(numpy.min(timer.repeat(num_repeats, 1)))
        sess.close()

        sess = tensorflow.Session()
        result = tensorflow.add(A, B)
        timer = timeit.Timer("sess.run(result)", setup="import tensorflow; from __main__ import sess, A, B, result")
        current_times.append(numpy.min(timer.repeat(num_repeats, 1)))
        sess.close()

        sess = tensorflow.Session()
        result = tensorflow.reduce_mean(A)
        timer = timeit.Timer("sess.run(result)", setup="import tensorflow; from __main__ import sess, A, B, result")
        current_times.append(numpy.min(timer.repeat(num_repeats, 1)))
        sess.close()

        sess = tensorflow.Session()
        result = tensorflow.reduce_min(A)
        timer = timeit.Timer("sess.run(result)", setup="import tensorflow; from __main__ import sess, A, B, result")
        current_times.append(numpy.min(timer.repeat(num_repeats, 1)))
        sess.close()

        times.append(current_times)

    times = numpy.array(times)
    print(times)
    numpy.save(data_dir + "tensorflow_times", times)

def plot_times():
    numpy_times = numpy.load(data_dir + "numpy_times.npy")
    theano_times = numpy.load(data_dir + "theano_times.npy")
    tensorflow_times = numpy.load(data_dir + "tensorflow_times.npy")

    titles = ["dot", "exp", "sum", "add", "mean", "min"]

    for i, title in enumerate(titles):
        plt.figure(i)
        plt.title(title)
        plt.plot(numpy_times[:, 0], numpy_times[:, i + 1], label="numpy")
        plt.plot(theano_times[:, 0], theano_times[:, i + 1], label="theano")
        plt.plot(tensorflow_times[:, 0], tensorflow_times[:, i + 1], label="tensorflow")
        plt.legend(loc="upper left")
        plt.xlabel("size")
        plt.ylabel("time (s)")
        plt.plot()

        # print(title)
        # print(numpy.c_[numpy_times[:, i + 1], theano_times[:, i + 1], tensorflow_times[:, i + 1]])

    plt.show()

if len(sys.argv) == 2 and sys.argv[1] == '1':
    benchmark_numpy()
elif len(sys.argv) == 2 and sys.argv[1] == '2':
    benchmark_theano()
elif len(sys.argv) == 2 and sys.argv[1] == '3':
    benchmark_tensorflow()
else:
    plot_times()
<commit_msg>Tidy up and check results<commit_after>import numpy
import timeit
import matplotlib.pyplot as plt
import sys
from tyrehug.settings import DATA_DIR, get_dir

numpy.random.seed(21)
num_repeats = 20
min_size = 500
max_size = 5001
step = 500

data_dir = get_dir(DATA_DIR)

def benchmark_numpy():
    times = []
    print("Benchmarking Numpy " + str(numpy.__version__))

    for i in range(min_size, max_size, step):
        print(i)
        global A, B
        A = numpy.random.rand(i, i).astype(numpy.float32)
        B = numpy.random.rand(i, i).astype(numpy.float32)

        current_times = [i]

        timer = timeit.Timer("numpy.dot(A, B)", "import numpy; from __main__ import A, B")
        current_times.append(numpy.min(timer.repeat(num_repeats, 1)))

        timer = timeit.Timer("numpy.exp(A)", "import numpy; from __main__ import A")
        current_times.append(numpy.min(timer.repeat(num_repeats, 1)))

        timer = timeit.Timer("numpy.sum(A)", "import numpy; from __main__ import A")
        current_times.append(numpy.min(timer.repeat(num_repeats, 1)))

        timer = timeit.Timer("A + B", "import numpy; from __main__ import A, B")
        current_times.append(numpy.min(timer.repeat(num_repeats, 1)))

        timer = timeit.Timer("numpy.mean(A)", "import numpy; from __main__ import A")
        current_times.append(numpy.min(timer.repeat(num_repeats, 1)))

        timer = timeit.Timer("numpy.min(A)", "import numpy; from __main__ import A")
        current_times.append(numpy.min(timer.repeat(num_repeats, 1)))

        times.append(current_times)

    times = numpy.array(times)
    filename = data_dir + "numpy_times"
    numpy.save(filename, times)
    print("Saved results as {}.npy".format(filename))

    print("\nResults check")
    print("-"*14)

    # Print results to compare them
    print("||dot(A, B)||={}".format(numpy.linalg.norm(numpy.dot(A, B))))
    print("||exp(A, B)||={}".format(numpy.linalg.norm(numpy.exp(A, B))))
    print("sum(A)={}".format(numpy.sum(A)))
    print("||A+B||={}".format(numpy.linalg.norm(A + B)))
    print("mean(A)={}".format(numpy.mean(A)))
    print("min(A)={}".format(numpy.min(A)))

def benchmark_theano():
    from theano import function
    import theano
    import theano.tensor as T
    print("Benchmarking Theano " + str(theano.__version__))

    times = []

    for i in range(min_size, max_size, step):
        print(i)
        global A, B
        A = numpy.random.rand(i, i).astype(numpy.float32)
        B = numpy.random.rand(i, i).astype(numpy.float32)
        current_times = [i]

        global f
        A = theano.shared(A)
        B = theano.shared(B)

        z = T.dot(A, B)
        f = function([], z)
        timer = timeit.Timer("f()", "from __main__ import f")
        current_times.append(numpy.min(timer.repeat(num_repeats, 1)))

        z = T.exp(A)
        f = function([], z)
        timer = timeit.Timer("f()", "from __main__ import f")
        current_times.append(numpy.min(timer.repeat(num_repeats, 1)))

        z = T.sum(A)
        f = function([], z)
        timer = timeit.Timer("f()", "from __main__ import f")
        current_times.append(numpy.min(timer.repeat(num_repeats, 1)))

        z = T.add(A, B)
        f = function([], z)
        timer = timeit.Timer("f()", "from __main__ import f")
        current_times.append(numpy.min(timer.repeat(num_repeats, 1)))

        z = T.mean(A)
        f = function([], z)
        timer = timeit.Timer("f()", "from __main__ import f")
        current_times.append(numpy.min(timer.repeat(num_repeats, 1)))

        z = T.min(A)
        f = function([], z)
        timer = timeit.Timer("f()", "from __main__ import f")
        current_times.append(numpy.min(timer.repeat(num_repeats, 1)))

        times.append(current_times)

    times = numpy.array(times)
    filename = data_dir + "theano_times"
    numpy.save(filename, times)
    print("Saved results as {}.npy".format(filename))

    print("\nResults check")
    print("-"*14)
    f = function([], T.dot(A, B))
    print("||dot(A, B)||={}".format(numpy.linalg.norm(f())))
    f = function([], T.exp(A))
    print("||exp(A)||={}".format(numpy.linalg.norm(f())))
    f = function([], T.sum(A))
    print("sum(A)={}".format(f()))
    f = function([], A+B)
    print("||A+B||={}".format(numpy.linalg.norm(f())))
    f = function([], T.mean(A))
    print("mean(A)={}".format(f()))
    f = function([], T.min(A))
    print("min(A)={}".format(f()))

def benchmark_tensorflow():
    import tensorflow
    print("Benchmarking Tensorflow " + str(tensorflow.__version__))

    times = []

    for i in range(min_size, max_size, step):
        print(i)
        global A, B
        A = numpy.random.rand(i, i).astype(numpy.float32)
        B = numpy.random.rand(i, i).astype(numpy.float32)
        current_times = [i]

        global f
        # Set up the tensorflow stuff
        A = tensorflow.constant(A)
        B = tensorflow.constant(B)
        global sess, result

        sess = tensorflow.Session()
        result = tensorflow.matmul(A, B)
        timer = timeit.Timer("sess.run(result)", setup="import tensorflow; from __main__ import sess, A, B, result")
        current_times.append(numpy.min(timer.repeat(num_repeats, 1)))
        sess.close()

        sess = tensorflow.Session()
        result = tensorflow.exp(A)
        timer = timeit.Timer("sess.run(result)", setup="import tensorflow; from __main__ import sess, A, B, result")
        current_times.append(numpy.min(timer.repeat(num_repeats, 1)))
        sess.close()

        sess = tensorflow.Session()
        result = tensorflow.reduce_sum(A)
        timer = timeit.Timer("sess.run(result)", setup="import tensorflow; from __main__ import sess, A, B, result")
        current_times.append(numpy.min(timer.repeat(num_repeats, 1)))
        sess.close()

        sess = tensorflow.Session()
        result = tensorflow.add(A, B)
        timer = timeit.Timer("sess.run(result)", setup="import tensorflow; from __main__ import sess, A, B, result")
        current_times.append(numpy.min(timer.repeat(num_repeats, 1)))
        sess.close()

        sess = tensorflow.Session()
        result = tensorflow.reduce_mean(A)
        timer = timeit.Timer("sess.run(result)", setup="import tensorflow; from __main__ import sess, A, B, result")
        current_times.append(numpy.min(timer.repeat(num_repeats, 1)))
        sess.close()

        sess = tensorflow.Session()
        result = tensorflow.reduce_min(A)
        timer = timeit.Timer("sess.run(result)", setup="import tensorflow; from __main__ import sess, A, B, result")
        current_times.append(numpy.min(timer.repeat(num_repeats, 1)))
        sess.close()

        times.append(current_times)

    times = numpy.array(times)
    filename = data_dir + "tensorflow_times"
    numpy.save(filename, times)
    print("Saved results as {}.npy".format(filename))

    print("\nResults check")
    print("-"*14)
    sess = tensorflow.Session()
    result = tensorflow.matmul(A, B)
    print("||dot(A, B)||={}".format(numpy.linalg.norm(sess.run(result))))
    result = tensorflow.exp(A)
    print("||exp(A)||={}".format(numpy.linalg.norm(sess.run(result))))
    result = tensorflow.reduce_sum(A)
    print("sum(A)={}".format(sess.run(result)))
    result = tensorflow.add(A, B)
    print("||A+B||={}".format(numpy.linalg.norm(sess.run(result))))
    result = tensorflow.reduce_mean(A)
    print("mean(A)={}".format(sess.run(result)))
    result = tensorflow.reduce_min(A)
    print("min(A)={}".format(sess.run(result)))
    sess.close()

def plot_times():
    numpy_times = numpy.load(data_dir + "numpy_times.npy")
    theano_times = numpy.load(data_dir + "theano_times.npy")
    tensorflow_times = numpy.load(data_dir + "tensorflow_times.npy")

    titles = ["dot", "exp", "sum", "add", "mean", "min"]

    for i, title in enumerate(titles):
        plt.figure(i)
        plt.title(title)
        plt.plot(numpy_times[:, 0], numpy_times[:, i + 1], label="numpy")
        plt.plot(theano_times[:, 0], theano_times[:, i + 1], label="theano")
        plt.plot(tensorflow_times[:, 0], tensorflow_times[:, i + 1], label="tensorflow")
        plt.legend(loc="upper left")
        plt.xlabel("size")
        plt.ylabel("time (s)")
        plt.plot()

        # print(title)
        print(numpy.c_[numpy_times[:, i + 1], theano_times[:, i + 1], tensorflow_times[:, i + 1]])

    plt.show()

if len(sys.argv) == 2 and sys.argv[1] == '1':
    benchmark_numpy()
elif len(sys.argv) == 2 and sys.argv[1] == '2':
    benchmark_theano()
elif len(sys.argv) == 2 and sys.argv[1] == '3':
    benchmark_tensorflow()
else:
    plot_times()

# Not sure why A+B results are different in numpy vs theano and tensorflow - could it be precision?
<|endoftext|>


<commit_before>import factory

class UCReportFactory(factory.DjangoModelFactory):
    rating = None
    last_updated = None
<commit_msg>Add FACTORY_FOR<commit_after>import factory

from ..models import UCRiskReport

class UCReportFactory(factory.DjangoModelFactory):
    FACTORY_FOR = UCRiskReport

    rating = None
    last_updated = None
<|endoftext|>


<commit_before>from __future__ import unicode_literals
import warnings
from django.core.exceptions import ImproperlyConfigured
from django.db.models.fields import FieldDoesNotExist
from mongoengine.errors import ValidationError
from rest_framework import serializers
from mongoengine import fields as me_fields
from django.core.paginator import Page
from django.db import models
from django.forms import widgets

from collections import OrderedDict
import copy

from rest_framework_mongoengine.utils import get_field_info

from .fields import ReferenceField, ListField, EmbeddedDocumentField, DynamicField, ObjectIdField
from rest_framework import fields as drf_fields
from rest_framework.utils.field_mapping import get_field_kwargs

class MongoEngineModelSerializer(serializers.ModelSerializer):
    """
    Model Serializer that supports Mongoengine
    """
    MAX_RECURSION_DEPTH = 5  # default value of depth

    field_mapping = {
        me_fields.FloatField: drf_fields.FloatField,
        me_fields.IntField: drf_fields.IntegerField,
        me_fields.DateTimeField: drf_fields.DateTimeField,
        me_fields.EmailField: drf_fields.EmailField,
        me_fields.URLField: drf_fields.URLField,
        me_fields.StringField: drf_fields.CharField,
        me_fields.BooleanField: drf_fields.BooleanField,
        me_fields.FileField: drf_fields.FileField,
        me_fields.ImageField: drf_fields.ImageField,
        me_fields.ObjectIdField: ObjectIdField,
        me_fields.ReferenceField: ReferenceField,
        me_fields.ListField: ListField,
        me_fields.EmbeddedDocumentField: EmbeddedDocumentField,
        me_fields.DynamicField: DynamicField,
        me_fields.DecimalField: drf_fields.DecimalField,
        me_fields.UUIDField: drf_fields.CharField
    }

    def get_validators(self):
        validators = getattr(getattr(self, 'Meta', None), 'validators', [])
        return validators

    # def perform_validation(self, attrs):
    #     """
    #     Rest Framework built-in validation + related model validations
    #     """
    #     for field_name, field in self.fields.items():
    #         if field_name in self._errors:
    #             continue
    #
    #         source = field.source or field_name
    #         if self.partial and source not in attrs:
    #             continue
    #
    #         if field_name in attrs and hasattr(field, 'model_field'):
    #             try:
    #                 field.model_field.validate(attrs[field_name])
    #             except ValidationError as err:
    #                 self._errors[field_name] = str(err)
    #
    #         try:
    #             validate_method = getattr(self, 'validate_%s' % field_name, None)
    #             if validate_method:
    #                 attrs = validate_method(attrs, source)
    #         except serializers.ValidationError as err:
    #             self._errors[field_name] = self._errors.get(field_name, []) + list(err.messages)
    #
    #     if not self._errors:
    #         try:
    #             attrs = self.validate(attrs)
    #         except serializers.ValidationError as err:
    #             if hasattr(err, 'message_dict'):
    #                 for field_name, error_messages in err.message_dict.items():
    #                     self._errors[field_name] = self._errors.get(field_name, []) + list(error_messages)
    #             elif hasattr(err, 'messages'):
    #                 self._errors['non_field_errors'] = err.messages
    #
    #     return attrs

    # def restore_object(self, attrs, instance=None):
    #     if instance is None:
    #         instance = self.opts.model()
    #
    #     dynamic_fields = self.get_dynamic_fields(instance)
    #     all_fields = dict(dynamic_fields, **self.fields)
    #
    #     for key, val in attrs.items():
    #         field = all_fields.get(key)
    #         if not field or field.read_only:
    #             continue
    #
    #         if isinstance(field, serializers.Serializer):
    #             many = field.many
    #
    #             def _restore(field, item):
    #                 # looks like a bug, sometimes there are decerialized objects in attrs
    #                 # sometimes they are just dicts
    #                 if isinstance(item, BaseDocument):
    #                     return item
    #                 return field.from_native(item)
    #
    #             if many:
    #                 val = [_restore(field, item) for item in val]
    #             else:
    #                 val = _restore(field, val)
    #
    #         key = getattr(field, 'source', None) or key
    #         try:
    #             setattr(instance, key, val)
    #         except ValueError:
    #             self._errors[key] = self.error_messages['required']
    #
    #     return instance

    def get_fields(self):
        declared_fields = copy.deepcopy(self._declared_fields)

        ret = OrderedDict()
        model = getattr(self.Meta, 'model')
        fields = getattr(self.Meta, 'fields', None)
        exclude = getattr(self.Meta, 'exclude', None)
        depth = getattr(self.Meta, 'depth', 0)
        extra_kwargs = getattr(self.Meta, 'extra_kwargs', {})

        if fields and not isinstance(fields, (list, tuple)):
            raise TypeError(
                'The `fields` option must be a list or tuple. Got %s.' %
                type(fields).__name__
            )

        if exclude and not isinstance(exclude, (list, tuple)):
            raise TypeError(
                'The `exclude` option must be a list or tuple. Got %s.' %
                type(exclude).__name__
            )

        assert not (fields and exclude), "Cannot set both 'fields' and 'exclude'."

        extra_kwargs = self._include_additional_options(extra_kwargs)

        # # Retrieve metadata about fields & relationships on the model class.
        info = get_field_info(model)

        # Use the default set of field names if none is supplied explicitly.
        if fields is None:
            fields = self._get_default_field_names(declared_fields, info)
            exclude = getattr(self.Meta, 'exclude', None)
            if exclude is not None:
                for field_name in exclude:
                    fields.remove(field_name)

        # Determine the set of model fields, and the fields that they map to.
        # We actually only need this to deal with the slightly awkward case
        # of supporting `unique_for_date`/`unique_for_month`/`unique_for_year`.
        model_field_mapping = {}
        for field_name in fields:
            if field_name in declared_fields:
                field = declared_fields[field_name]
                source = field.source or field_name
            else:
                try:
                    source = extra_kwargs[field_name]['source']
                except KeyError:
                    source = field_name
            # Model fields will always have a simple source mapping,
            # they can't be nested attribute lookups.
            if '.' not in source and source != '*':
                model_field_mapping[source] = field_name

        # Determine if we need any additional `HiddenField` or extra keyword
        # arguments to deal with `unique_for` dates that are required to
        # be in the input data in order to validate it.
        hidden_fields = {}

        # Now determine the fields that should be included on the serializer.
        for field_name in fields:
            if field_name in declared_fields:
                # Field is explicitly declared on the class, use that.
                ret[field_name] = declared_fields[field_name]
                continue

            elif field_name in info.fields_and_pk:
                # Create regular model fields.
                model_field = info.fields_and_pk[field_name]
                field_cls = self.field_mapping[model_field.__class__]
                kwargs = self.get_field_kwargs(model_field)
                if 'choices' in kwargs:
                    # Fields with choices get coerced into `ChoiceField`
                    # instead of using their regular typed field.
                    field_cls = drf_fields.ChoiceField
                if not issubclass(field_cls, drf_fields.CharField) and not issubclass(field_cls, drf_fields.ChoiceField):
                    # `allow_blank` is only valid for textual fields.
                    kwargs.pop('allow_blank', None)

            elif hasattr(model, field_name):
                # Create a read only field for model methods and properties.
                field_cls = drf_fields.ReadOnlyField
                kwargs = {}

            else:
                raise ImproperlyConfigured(
                    'Field name `%s` is not valid for model `%s`.' %
                    (field_name, model.__class__.__name__)
                )

            # Check that any fields declared on the class are
            # also explicitly included in `Meta.fields`.
            missing_fields = set(declared_fields.keys()) - set(fields)
            if missing_fields:
                missing_field = list(missing_fields)[0]
                raise ImproperlyConfigured(
                    'Field `%s` has been declared on serializer `%s`, but '
                    'is missing from `Meta.fields`.' %
                    (missing_field, self.__class__.__name__)
                )

            # Populate any kwargs defined in `Meta.extra_kwargs`
            extras = extra_kwargs.get(field_name, {})
            if extras.get('read_only', False):
                for attr in [
                    'required', 'default', 'allow_blank', 'allow_null',
                    'min_length', 'max_length', 'min_value', 'max_value',
                    'validators', 'queryset'
                ]:
                    kwargs.pop(attr, None)

            if extras.get('default') and kwargs.get('required') is False:
                kwargs.pop('required')

            kwargs.update(extras)

            # Create the serializer field.
            ret[field_name] = field_cls(**kwargs)

        for field_name, field in hidden_fields.items():
            ret[field_name] = field

        return ret

    def get_dynamic_fields(self, document):
        dynamic_fields = {}
        if document is not None and document._dynamic:
            for name, field in document._dynamic_fields.items():
                dynamic_fields[name] = DynamicField(field_name=name, source=name, **self.get_field_kwargs(field))
        return dynamic_fields

    def get_field_kwargs(self, model_field):
        kwargs = {}

        if type(model_field) in (me_fields.ReferenceField, me_fields.EmbeddedDocumentField,
                                     me_fields.ListField, me_fields.DynamicField):
            kwargs['model_field'] = model_field
            kwargs['depth'] = getattr(self.Meta, 'depth', self.MAX_RECURSION_DEPTH)

        if type(model_field) is me_fields.ObjectIdField:
            kwargs['required'] = False
        else:
            kwargs['required'] = model_field.required

        if type(model_field) is me_fields.EmbeddedDocumentField:
            kwargs['document_type'] = model_field.document_type

        if model_field.default:
            kwargs['required'] = False
            kwargs['default'] = model_field.default

        if model_field.__class__ == models.TextField:
            kwargs['widget'] = widgets.Textarea

        attribute_dict = {
            me_fields.StringField: ['max_length'],
            me_fields.DecimalField: ['min_value', 'max_value'],
            me_fields.EmailField: ['max_length'],
            me_fields.FileField: ['max_length'],
            me_fields.URLField: ['max_length'],
        }

        if model_field.__class__ in attribute_dict:
            attributes = attribute_dict[model_field.__class__]
            for attribute in attributes:
                kwargs.update({attribute: getattr(model_field, attribute)})

        return kwargs

    def create(self, validated_data):
        serializers.raise_errors_on_nested_writes('create', self, validated_data)

        ModelClass = self.Meta.model
        try:
            instance = ModelClass.objects.create(**validated_data)
        except TypeError as exc:
            msg = (
                'Got a `TypeError` when calling `%s.objects.create()`. '
                'This may be because you have a writable field on the '
                'serializer class that is not a valid argument to '
                '`%s.objects.create()`. You may need to make the field '
                'read-only, or override the %s.create() method to handle '
                'this correctly.\nOriginal exception text was: %s.' %
                (
                    ModelClass.__name__,
                    ModelClass.__name__,
                    self.__class__.__name__,
                    exc
                )
            )
            raise TypeError(msg)

        return instance

    def to_representation(self, instance):
        """
        Object instance -> Dict of primitive datatypes.
        Serialize regular + dynamic fields
        """
        ret = OrderedDict()
        fields = [field for field in self.fields.values() if not field.write_only]
        fields += self.get_dynamic_fields(instance).values()

        for field in fields:
            attribute = field.get_attribute(instance)
            if attribute is None:
                ret[field.field_name] = None
            else:
                ret[field.field_name] = field.to_representation(attribute)

        return ret

    def to_internal_value(self, data):
        """
        Dict of native values <- Dict of primitive datatypes.
        """
        ret = super(MongoEngineModelSerializer, self).to_internal_value(data)
        [drf_fields.set_value(ret, [k], data[k]) for k in data if k not in ret]
        return ret

    # @property
    # def data(self):
    #     """
    #     Returns the serialized data on the serializer.
    #     """
    #     if self._data is None:
    #         obj = self.object
    #
    #         if self.many is not None:
    #             many = self.many
    #         else:
    #             many = hasattr(obj, '__iter__') and not isinstance(obj, (mongoengine.BaseDocument, Page, dict))
    #             if many:
    #                 warnings.warn('Implicit list/queryset serialization is deprecated. '
    #                               'Use the `many=True` flag when instantiating the serializer.',
    #                               DeprecationWarning, stacklevel=2)
    #
    #         if many:
    #             self._data = [self.to_native(item) for item in obj]
    #         else:
    #             self._data = self.to_native(obj)
    #
    #     return self._data<commit_msg>provide ValidationError for mongoengine errors<commit_after>from __future__ import unicode_literals
import warnings
from django.core.exceptions import ImproperlyConfigured
from django.db.models.fields import FieldDoesNotExist
from mongoengine.errors import ValidationError as me_ValidationError
from rest_framework import serializers
from mongoengine import fields as me_fields
from django.core.paginator import Page
from django.db import models
from django.forms import widgets

from collections import OrderedDict
import copy

from rest_framework_mongoengine.utils import get_field_info

from .fields import ReferenceField, ListField, EmbeddedDocumentField, DynamicField, ObjectIdField
from rest_framework import fields as drf_fields

class MongoEngineModelSerializer(serializers.ModelSerializer):
    """
    Model Serializer that supports Mongoengine
    """
    MAX_RECURSION_DEPTH = 5  # default value of depth

    field_mapping = {
        me_fields.FloatField: drf_fields.FloatField,
        me_fields.IntField: drf_fields.IntegerField,
        me_fields.DateTimeField: drf_fields.DateTimeField,
        me_fields.EmailField: drf_fields.EmailField,
        me_fields.URLField: drf_fields.URLField,
        me_fields.StringField: drf_fields.CharField,
        me_fields.BooleanField: drf_fields.BooleanField,
        me_fields.FileField: drf_fields.FileField,
        me_fields.ImageField: drf_fields.ImageField,
        me_fields.ObjectIdField: ObjectIdField,
        me_fields.ReferenceField: ReferenceField,
        me_fields.ListField: ListField,
        me_fields.EmbeddedDocumentField: EmbeddedDocumentField,
        me_fields.DynamicField: DynamicField,
        me_fields.DecimalField: drf_fields.DecimalField,
        me_fields.UUIDField: drf_fields.CharField
    }

    def get_validators(self):
        validators = getattr(getattr(self, 'Meta', None), 'validators', [])
        return validators

    # def run_validation(self, data=empty):
    #     """
    #     Rest Framework built-in validation + related model validations
    #     """
    #     for field_name, field in self.fields.items():
    #         if field_name in self._errors:
    #             continue
    #
    #         source = field.source or field_name
    #         if self.partial and source not in attrs:
    #             continue
    #
    #         if field_name in attrs and hasattr(field, 'model_field'):
    #             try:
    #                 field.model_field.validate(attrs[field_name])
    #             except ValidationError as err:
    #                 self._errors[field_name] = str(err)
    #
    #         try:
    #             validate_method = getattr(self, 'validate_%s' % field_name, None)
    #             if validate_method:
    #                 attrs = validate_method(attrs, source)
    #         except serializers.ValidationError as err:
    #             self._errors[field_name] = self._errors.get(field_name, []) + list(err.messages)
    #
    #     if not self._errors:
    #         try:
    #             attrs = self.validate(attrs)
    #         except serializers.ValidationError as err:
    #             if hasattr(err, 'message_dict'):
    #                 for field_name, error_messages in err.message_dict.items():
    #                     self._errors[field_name] = self._errors.get(field_name, []) + list(error_messages)
    #             elif hasattr(err, 'messages'):
    #                 self._errors['non_field_errors'] = err.messages
    #
    #     return attrs

    # def validate(self, attrs):
    #     return self.Meta.model.validate(attrs)

    def get_fields(self):
        declared_fields = copy.deepcopy(self._declared_fields)

        ret = OrderedDict()
        model = getattr(self.Meta, 'model')
        fields = getattr(self.Meta, 'fields', None)
        exclude = getattr(self.Meta, 'exclude', None)
        depth = getattr(self.Meta, 'depth', 0)
        extra_kwargs = getattr(self.Meta, 'extra_kwargs', {})

        if fields and not isinstance(fields, (list, tuple)):
            raise TypeError(
                'The `fields` option must be a list or tuple. Got %s.' %
                type(fields).__name__
            )

        if exclude and not isinstance(exclude, (list, tuple)):
            raise TypeError(
                'The `exclude` option must be a list or tuple. Got %s.' %
                type(exclude).__name__
            )

        assert not (fields and exclude), "Cannot set both 'fields' and 'exclude'."

        extra_kwargs = self._include_additional_options(extra_kwargs)

        # # Retrieve metadata about fields & relationships on the model class.
        info = get_field_info(model)

        # Use the default set of field names if none is supplied explicitly.
        if fields is None:
            fields = self._get_default_field_names(declared_fields, info)
            exclude = getattr(self.Meta, 'exclude', None)
            if exclude is not None:
                for field_name in exclude:
                    fields.remove(field_name)

        # Determine the set of model fields, and the fields that they map to.
        # We actually only need this to deal with the slightly awkward case
        # of supporting `unique_for_date`/`unique_for_month`/`unique_for_year`.
        model_field_mapping = {}
        for field_name in fields:
            if field_name in declared_fields:
                field = declared_fields[field_name]
                source = field.source or field_name
            else:
                try:
                    source = extra_kwargs[field_name]['source']
                except KeyError:
                    source = field_name
            # Model fields will always have a simple source mapping,
            # they can't be nested attribute lookups.
            if '.' not in source and source != '*':
                model_field_mapping[source] = field_name

        # Determine if we need any additional `HiddenField` or extra keyword
        # arguments to deal with `unique_for` dates that are required to
        # be in the input data in order to validate it.
        hidden_fields = {}

        # Now determine the fields that should be included on the serializer.
        for field_name in fields:
            if field_name in declared_fields:
                # Field is explicitly declared on the class, use that.
                ret[field_name] = declared_fields[field_name]
                continue

            elif field_name in info.fields_and_pk:
                # Create regular model fields.
                model_field = info.fields_and_pk[field_name]
                field_cls = self.field_mapping[model_field.__class__]
                kwargs = self.get_field_kwargs(model_field)
                if 'choices' in kwargs:
                    # Fields with choices get coerced into `ChoiceField`
                    # instead of using their regular typed field.
                    field_cls = drf_fields.ChoiceField
                if not issubclass(field_cls, drf_fields.CharField) and not issubclass(field_cls, drf_fields.ChoiceField):
                    # `allow_blank` is only valid for textual fields.
                    kwargs.pop('allow_blank', None)

            elif hasattr(model, field_name):
                # Create a read only field for model methods and properties.
                field_cls = drf_fields.ReadOnlyField
                kwargs = {}

            else:
                raise ImproperlyConfigured(
                    'Field name `%s` is not valid for model `%s`.' %
                    (field_name, model.__class__.__name__)
                )

            # Check that any fields declared on the class are
            # also explicitly included in `Meta.fields`.
            missing_fields = set(declared_fields.keys()) - set(fields)
            if missing_fields:
                missing_field = list(missing_fields)[0]
                raise ImproperlyConfigured(
                    'Field `%s` has been declared on serializer `%s`, but '
                    'is missing from `Meta.fields`.' %
                    (missing_field, self.__class__.__name__)
                )

            # Populate any kwargs defined in `Meta.extra_kwargs`
            extras = extra_kwargs.get(field_name, {})
            if extras.get('read_only', False):
                for attr in [
                    'required', 'default', 'allow_blank', 'allow_null',
                    'min_length', 'max_length', 'min_value', 'max_value',
                    'validators', 'queryset'
                ]:
                    kwargs.pop(attr, None)

            if extras.get('default') and kwargs.get('required') is False:
                kwargs.pop('required')

            kwargs.update(extras)

            # Create the serializer field.
            ret[field_name] = field_cls(**kwargs)

        for field_name, field in hidden_fields.items():
            ret[field_name] = field

        return ret

    def get_dynamic_fields(self, document):
        dynamic_fields = {}
        if document is not None and document._dynamic:
            for name, field in document._dynamic_fields.items():
                dynamic_fields[name] = DynamicField(field_name=name, source=name, **self.get_field_kwargs(field))
        return dynamic_fields

    def get_field_kwargs(self, model_field):
        kwargs = {}

        if type(model_field) in (me_fields.ReferenceField, me_fields.EmbeddedDocumentField,
                                     me_fields.ListField, me_fields.DynamicField):
            kwargs['model_field'] = model_field
            kwargs['depth'] = getattr(self.Meta, 'depth', self.MAX_RECURSION_DEPTH)

        if type(model_field) is me_fields.ObjectIdField:
            kwargs['required'] = False
        else:
            kwargs['required'] = model_field.required

        if type(model_field) is me_fields.EmbeddedDocumentField:
            kwargs['document_type'] = model_field.document_type

        if model_field.default:
            kwargs['required'] = False
            kwargs['default'] = model_field.default

        if model_field.__class__ == models.TextField:
            kwargs['widget'] = widgets.Textarea

        attribute_dict = {
            me_fields.StringField: ['max_length'],
            me_fields.DecimalField: ['min_value', 'max_value'],
            me_fields.EmailField: ['max_length'],
            me_fields.FileField: ['max_length'],
            me_fields.URLField: ['max_length'],
        }

        if model_field.__class__ in attribute_dict:
            attributes = attribute_dict[model_field.__class__]
            for attribute in attributes:
                kwargs.update({attribute: getattr(model_field, attribute)})

        return kwargs

    def create(self, validated_data):
        serializers.raise_errors_on_nested_writes('create', self, validated_data)

        ModelClass = self.Meta.model
        try:
            instance = ModelClass.objects.create(**validated_data)
        except TypeError as exc:
            msg = (
                'Got a `TypeError` when calling `%s.objects.create()`. '
                'This may be because you have a writable field on the '
                'serializer class that is not a valid argument to '
                '`%s.objects.create()`. You may need to make the field '
                'read-only, or override the %s.create() method to handle '
                'this correctly.\nOriginal exception text was: %s.' %
                (
                    ModelClass.__name__,
                    ModelClass.__name__,
                    self.__class__.__name__,
                    exc
                )
            )
            raise TypeError(msg)
        except me_ValidationError as exc:
            msg = (
                'Got a `ValidationError` when calling `%s.objects.create()`. '
                'This may be because request data satisfies serializer validations '
                'but not Mongoengine`s. You may need to check consistency between '
                '%s and %s.\nIf that is not the case, please open a ticket '
                'regarding this issue on https://github.com/umutbozkurt/django-rest-framework-mongoengine/issues'
                '\nOriginal exception was: %s' %
                (
                    ModelClass.__name__,
                    ModelClass.__name__,
                    self.__class__.__name__,
                    exc
                )
            )
            raise me_ValidationError(msg)

        return instance

    def to_representation(self, instance):
        """
        Object instance -> Dict of primitive datatypes.
        Serialize regular + dynamic fields
        """
        ret = OrderedDict()
        fields = [field for field in self.fields.values() if not field.write_only]
        fields += self.get_dynamic_fields(instance).values()

        for field in fields:
            attribute = field.get_attribute(instance)
            if attribute is None:
                ret[field.field_name] = None
            else:
                ret[field.field_name] = field.to_representation(attribute)

        return ret

    def to_internal_value(self, data):
        """
        Dict of native values <- Dict of primitive datatypes.
        """
        ret = super(MongoEngineModelSerializer, self).to_internal_value(data)
        [drf_fields.set_value(ret, [k], data[k]) for k in data if k not in ret]
        return ret

    # @property
    # def data(self):
    #     """
    #     Returns the serialized data on the serializer.
    #     """
    #     if self._data is None:
    #         obj = self.object
    #
    #         if self.many is not None:
    #             many = self.many
    #         else:
    #             many = hasattr(obj, '__iter__') and not isinstance(obj, (mongoengine.BaseDocument, Page, dict))
    #             if many:
    #                 warnings.warn('Implicit list/queryset serialization is deprecated. '
    #                               'Use the `many=True` flag when instantiating the serializer.',
    #                               DeprecationWarning, stacklevel=2)
    #
    #         if many:
    #             self._data = [self.to_native(item) for item in obj]
    #         else:
    #             self._data = self.to_native(obj)
    #
    #     return self._data<|endoftext|>


<commit_before>import sys
import string
import json
import time

from urllib import unquote
from flask import request, redirect, render_template

from . import index
from ..utils import GenerateTraceURLId
from .. import db

@index.route('/', methods=['GET'])
def index():
    # read GET values
    spanName = request.args.get('spanName')

    lastServiceName = request.cookies.get('last-serviceName')
    serviceName = request.args.get('serviceName') or '' if lastServiceName is None else unquote(lastServiceName)

    timestamp = request.args.get('timestamp')
    limit = request.args.get('limit') or 10

    formSubmitted = True

    if timestamp is None or timestamp.strip() == '':
        formSubmitted = False
        timestamp = int(time.time() * 1000000)

    # get database engine connection
    connection = db.engine.connect()

    # query results
    traceResults = None

    # query database based on query parameters if service is given
    if formSubmitted:
        # query results that would be sent over to view
        traceResults = []

        # find all traces to which related to this service
        query = "SELECT DISTINCT trace_id \
                FROM zipnish_annotations "

        # where
        whereQuery = ''

        if serviceName is not None and len(serviceName) > 0:
            whereQuery += " service_name = '%s' " % serviceName

        if spanName is not None and len(spanName) > 0 and spanName != 'all':
            if len(whereQuery) > 0:
                whereQuery += " AND "
            whereQuery += " span_name = '%s' " % spanName

        if timestamp is not None and len(timestamp) > 0:
            if len(whereQuery) > 0:
                whereQuery += " AND "
            whereQuery += " a_timestamp < %s " % timestamp

        # attach where clause only if there is a criteria
        if len(whereQuery) > 0:
            whereQuery = " WHERE %s " % whereQuery
            query += whereQuery

        # order by
        orderByQuery = " ORDER BY a_timestamp DESC"
        query += orderByQuery

        # limit search results
        limitQuery = ""

        if limit is not None:
            limitQuery += " LIMIT 0, %s" % limit
            query += limitQuery

        traceIds = []
        resultTraceIds = connection.execute(query)

        for row in resultTraceIds:
            traceIds.append(row['trace_id'])

        if len(traceIds) > 0:
            # find the number of DISTINCT spans, that above service connects with
            query = "SELECT COUNT(DISTINCT span_id) as spanCount, parent_id, created_ts, trace_id \
                    FROM zipnish_spans \
                    GROUP BY trace_id \
                    HAVING \
                    trace_id IN (%s) \
                    ORDER BY created_ts DESC" \
                    % (",".join(str(traceId) for traceId in traceIds))
            result = connection.execute(query)

            for row in result:
                trace = {}

                trace['serviceName'] = serviceName
                trace['spanCount'] = row['spanCount']
                trace['trace_id_long'] = row['trace_id']
                trace['trace_id'] = GenerateTraceURLId(row['trace_id'])

                startTime = (int(row['created_ts']) / 1000000)
                trace['startTime'] = time.strftime('%m-%d-%YT%H:%M:%S%z', time.gmtime(startTime))

                servicesQuery = "SELECT service_name, `value`, a_timestamp \
                        FROM zipnish_annotations \
                        WHERE trace_id = %s AND \
                        `value` IN ('cs', 'sr', 'ss', 'cr') \
                        ORDER BY service_name ASC" % (row['trace_id'])
                servicesResult = connection.execute(servicesQuery)

                services = {}
                service = None

                for serviceRow in servicesResult:
                    if serviceRow['service_name'] not in services:
                        services[serviceRow['service_name']] = {}
                        service = services[serviceRow['service_name']]
                        service['count'] = 0

                    if serviceRow['value'] == 'sr':
                        service['count'] += 1

                    service[serviceRow['value']] = serviceRow['a_timestamp']

                duration = 0
                serviceDuration = 0
                serviceDurations = []

                minTimestamp = sys.maxint
                maxTimestamp = 0

                selectedServiceDuration = 0
                totalTraceDuration = 0

                for key in services:
                    service = services[key]
                    if 'cs' in service:
                        minTimestamp = min(service['cr'], minTimestamp)
                        maxTimestamp = max(service['cs'], maxTimestamp)

                        serviceDuration = service['cr'] - service['cs']
                    else:
                        minTimestamp = min(service['sr'], minTimestamp)
                        maxTimestamp = max(service['ss'], maxTimestamp)

                        serviceDuration = service['ss'] - service['sr']

                    if serviceName == key:
                        selectedServiceDuration = serviceDuration

                    totalTraceDuration = max(totalTraceDuration, serviceDuration)

                    # service duration
                    serviceDurations.append({
                            'name': key,
                            'count': service['count'],
                            'duration': serviceDuration
                        })

                # total duration for a trace
                trace['duration'] = selectedServiceDuration

                # service durations

                # sort service durations
                serviceDurations = sorted(serviceDurations, key=lambda x: x['name'])

                trace['serviceDurations'] = serviceDurations

                #trace['serviceTimestampMin'] = minTimestamp
                #trace['serviceTimestampMax'] = maxTimestamp

                trace['servicesTotalDuration'] = '{:.3f}'.format(totalTraceDuration / 1000000)

                selectedServicePercentage = float(float(selectedServiceDuration) / float(totalTraceDuration)) * 100.0
                trace['selectedServicePercentage'] = '{:.2f}'.format(selectedServicePercentage)

                traceResults.append( trace )

        #return json.dumps(traceResults)

    # populate services
    services = []
    result = connection.execute("SELECT DISTINCT service_name FROM zipnish_annotations")

    for row in result:
        services.append( row['service_name'] )

    spans = []

    if serviceName:
        query = "SELECT DISTINCT span_name FROM zipnish_annotations WHERE service_name='%s'" % serviceName
        result = connection.execute(query)

        for row in result:
            spans.append( row['span_name'] )

    if len(spans) > 0:
        spans.insert(0, 'all')

    # close connection
    connection.close()

    return render_template('index.html', \
            results=traceResults, \
            services=services, spans=spans, \
            get_SpanName=spanName, get_ServiceName=serviceName, \
            get_Timestamp=timestamp, get_Limit=limit)
<commit_msg>fix floating point precision<commit_after>import sys
import string
import json
import time

from urllib import unquote
from flask import request, redirect, render_template

from . import index
from ..utils import GenerateTraceURLId
from .. import db

@index.route('/', methods=['GET'])
def index():
    # read GET values
    spanName = request.args.get('spanName')

    lastServiceName = request.cookies.get('last-serviceName')
    serviceName = request.args.get('serviceName') or '' if lastServiceName is None else unquote(lastServiceName)

    timestamp = request.args.get('timestamp')
    limit = request.args.get('limit') or 10

    formSubmitted = True

    if timestamp is None or timestamp.strip() == '':
        formSubmitted = False
        timestamp = int(time.time() * 1000000)

    # get database engine connection
    connection = db.engine.connect()

    # query results
    traceResults = None

    # query database based on query parameters if service is given
    if formSubmitted:
        # query results that would be sent over to view
        traceResults = []

        # find all traces to which related to this service
        query = "SELECT DISTINCT trace_id \
                FROM zipnish_annotations "

        # where
        whereQuery = ''

        if serviceName is not None and len(serviceName) > 0:
            whereQuery += " service_name = '%s' " % serviceName

        if spanName is not None and len(spanName) > 0 and spanName != 'all':
            if len(whereQuery) > 0:
                whereQuery += " AND "
            whereQuery += " span_name = '%s' " % spanName

        if timestamp is not None and len(timestamp) > 0:
            if len(whereQuery) > 0:
                whereQuery += " AND "
            whereQuery += " a_timestamp < %s " % timestamp

        # attach where clause only if there is a criteria
        if len(whereQuery) > 0:
            whereQuery = " WHERE %s " % whereQuery
            query += whereQuery

        # order by
        orderByQuery = " ORDER BY a_timestamp DESC"
        query += orderByQuery

        # limit search results
        limitQuery = ""

        if limit is not None:
            limitQuery += " LIMIT 0, %s" % limit
            query += limitQuery

        traceIds = []
        resultTraceIds = connection.execute(query)

        for row in resultTraceIds:
            traceIds.append(row['trace_id'])

        if len(traceIds) > 0:
            # find the number of DISTINCT spans, that above service connects with
            query = "SELECT COUNT(DISTINCT span_id) as spanCount, parent_id, created_ts, trace_id \
                    FROM zipnish_spans \
                    GROUP BY trace_id \
                    HAVING \
                    trace_id IN (%s) \
                    ORDER BY created_ts DESC" \
                    % (",".join(str(traceId) for traceId in traceIds))
            result = connection.execute(query)

            for row in result:
                trace = {}

                trace['serviceName'] = serviceName
                trace['spanCount'] = row['spanCount']
                trace['trace_id_long'] = row['trace_id']
                trace['trace_id'] = GenerateTraceURLId(row['trace_id'])

                startTime = (int(row['created_ts']) / 1000000)
                trace['startTime'] = time.strftime('%m-%d-%YT%H:%M:%S%z', time.gmtime(startTime))

                servicesQuery = "SELECT service_name, `value`, a_timestamp \
                        FROM zipnish_annotations \
                        WHERE trace_id = %s AND \
                        `value` IN ('cs', 'sr', 'ss', 'cr') \
                        ORDER BY service_name ASC" % (row['trace_id'])
                servicesResult = connection.execute(servicesQuery)

                services = {}
                service = None

                for serviceRow in servicesResult:
                    if serviceRow['service_name'] not in services:
                        services[serviceRow['service_name']] = {}
                        service = services[serviceRow['service_name']]
                        service['count'] = 0

                    if serviceRow['value'] == 'sr':
                        service['count'] += 1

                    service[serviceRow['value']] = serviceRow['a_timestamp']

                duration = 0
                serviceDuration = 0
                serviceDurations = []

                minTimestamp = sys.maxint
                maxTimestamp = 0

                selectedServiceDuration = 0
                totalTraceDuration = 0

                for key in services:
                    service = services[key]
                    if 'cs' in service:
                        minTimestamp = min(service['cr'], minTimestamp)
                        maxTimestamp = max(service['cs'], maxTimestamp)

                        serviceDuration = service['cr'] - service['cs']
                    else:
                        minTimestamp = min(service['sr'], minTimestamp)
                        maxTimestamp = max(service['ss'], maxTimestamp)

                        serviceDuration = service['ss'] - service['sr']

                    if serviceName == key:
                        selectedServiceDuration = serviceDuration

                    totalTraceDuration = max(totalTraceDuration, serviceDuration)

                    # service duration
                    serviceDurations.append({
                            'name': key,
                            'count': service['count'],
                            'duration': serviceDuration
                        })

                # total duration for a trace
                trace['duration'] = selectedServiceDuration

                # service durations

                # sort service durations
                serviceDurations = sorted(serviceDurations, key=lambda x: x['name'])

                trace['serviceDurations'] = serviceDurations

                #trace['serviceTimestampMin'] = minTimestamp
                #trace['serviceTimestampMax'] = maxTimestamp

                trace['servicesTotalDuration'] = '{:.3f}'.format(totalTraceDuration / 1000000.0)

                selectedServicePercentage = float(float(selectedServiceDuration) / float(totalTraceDuration)) * 100.0
                trace['selectedServicePercentage'] = '{:.2f}'.format(selectedServicePercentage)

                traceResults.append( trace )

        #return json.dumps(traceResults)

    # populate services
    services = []
    result = connection.execute("SELECT DISTINCT service_name FROM zipnish_annotations")

    for row in result:
        services.append( row['service_name'] )

    spans = []

    if serviceName:
        query = "SELECT DISTINCT span_name FROM zipnish_annotations WHERE service_name='%s'" % serviceName
        result = connection.execute(query)

        for row in result:
            spans.append( row['span_name'] )

    if len(spans) > 0:
        spans.insert(0, 'all')

    # close connection
    connection.close()

    return render_template('index.html', \
            results=traceResults, \
            services=services, spans=spans, \
            get_SpanName=spanName, get_ServiceName=serviceName, \
            get_Timestamp=timestamp, get_Limit=limit)
<|endoftext|>


<commit_before>import json
import time

from urllib import unquote
from flask import request, redirect, render_template

from . import index
from .. import db

@index.route('/', methods=['GET'])
def index():
    # read GET values
    spanName = request.args.get('spanName')
    serviceName = request.args.get('serviceName') or unquote(request.cookies.get('last-serviceName'))
    timestamp = request.args.get('timestamp')
    limit = request.args.get('limit') or 10

    formSubmitted = True

    if timestamp is None or timestamp.strip() == '':
        formSubmitted = False
        timestamp = int(time.time() * 1000000)

    # get database engine connection
    connection = db.engine.connect()

    # query results
    traceResults = None

    # query database based on query parameters if service is given
    if formSubmitted:
        # query results that would be sent over to view
        traceResults = []

        # find all traces to which related to this service
        query = "SELECT DISTINCT trace_id \
                FROM zipkin_annotations \
                WHERE service_name = '%s'" % (serviceName)

        traceIds = []
        resultTraceIds = connection.execute(query)

        for row in resultTraceIds:
            traceIds.append(row['trace_id'])

        if len(traceIds) > 0:
            # find the number of DISTINCT spans, that above service connects with
            query = "SELECT COUNT(DISTINCT span_id) as spanCount, parent_id, created_ts, trace_id \
                    FROM zipkin_spans \
                    GROUP BY trace_id \
                    HAVING \
                    trace_id IN (%s) \
                    ORDER BY created_ts DESC " \
                    % (",".join(str(traceId) for traceId in traceIds))
            result = connection.execute(query)

            for row in result:
                trace = {}

                trace['serviceName'] = serviceName
                trace['spanCount'] = row['spanCount']
                trace['trace_id'] = row['trace_id']

                startTime = (int(row['created_ts']) / 1000000)
                trace['startTime'] = time.strftime('%m-%d-%YT%H:%M:%S', time.gmtime(startTime))

                servicesQuery = "SELECT service_name, `value`, a_timestamp \
                        FROM zipkin_annotations \
                        WHERE trace_id = %s AND \
                        `value` IN ('cs', 'sr', 'ss', 'cr') \
                        ORDER BY service_name ASC" % (row['trace_id'])
                servicesResult = connection.execute(servicesQuery)

                services = {}
                service = None
                for serviceRow in servicesResult:
                    if serviceRow['service_name'] not in services:
                        services[serviceRow['service_name']] = {}
                        service = services[serviceRow['service_name']]

                    service[serviceRow['value']] = serviceRow['a_timestamp']

                duration = 0
                serviceDuration = 0
                serviceDurations = []

                for key in services:
                    service = services[key]
                    if 'cs' in service:
                        serviceDuration = service['cr'] - service['cs']
                    else:
                        serviceDuration = service['ss'] - service['sr']

                    # service duration
                    serviceDurations.append({
                            'name': key,
                            'duration': serviceDuration
                        })

                    # adding up duration to get total duration time
                    duration = duration + serviceDuration

                # total duration for a trace
                trace['duration'] = duration

                # service durations

                # sort service durations
                serviceDurations = sorted(serviceDurations, key=lambda x: x['name'])

                trace['serviceDurations'] = serviceDurations

                traceResults.append( trace )

        #return json.dumps(traceResults)

    # populate services
    services = []
    result = connection.execute("SELECT DISTINCT service_name FROM zipkin_annotations")

    for row in result:
        services.append( row['service_name'] )

    spans = []

    if serviceName:
        query = "SELECT DISTINCT span_name FROM zipkin_annotations WHERE service_name='%s'" % serviceName
        result = connection.execute(query)

        for row in result:
            spans.append( row['span_name'] )

    if len(spans) > 0:
        spans.insert(0, 'all')

    # close connection
    connection.close()

    return render_template('index.html', \
            results=traceResults, \
            services=services, spans=spans, \
            get_SpanName=spanName, get_ServiceName=serviceName, \
            get_Timestamp=timestamp, get_Limit=limit)
<commit_msg>add limit to querying annotation, and order service_name in desc order based on timestamp value<commit_after>import json
import time

from urllib import unquote
from flask import request, redirect, render_template

from . import index
from .. import db

@index.route('/', methods=['GET'])
def index():
    # read GET values
    spanName = request.args.get('spanName')
    serviceName = request.args.get('serviceName') or unquote(request.cookies.get('last-serviceName'))
    timestamp = request.args.get('timestamp')
    limit = request.args.get('limit') or 10

    formSubmitted = True

    if timestamp is None or timestamp.strip() == '':
        formSubmitted = False
        timestamp = int(time.time() * 1000000)

    # get database engine connection
    connection = db.engine.connect()

    # query results
    traceResults = None

    # query database based on query parameters if service is given
    if formSubmitted:
        # query results that would be sent over to view
        traceResults = []

        # find all traces to which related to this service
        query = "SELECT DISTINCT trace_id \
                FROM zipkin_annotations \
                WHERE service_name = '%s' \
                ORDER BY a_timestamp DESC \
                LIMIT 0, %s" % (serviceName, limit)

        traceIds = []
        resultTraceIds = connection.execute(query)

        for row in resultTraceIds:
            traceIds.append(row['trace_id'])

        if len(traceIds) > 0:
            # find the number of DISTINCT spans, that above service connects with
            query = "SELECT COUNT(DISTINCT span_id) as spanCount, parent_id, created_ts, trace_id \
                    FROM zipkin_spans \
                    GROUP BY trace_id \
                    HAVING \
                    trace_id IN (%s)" \
                    % (",".join(str(traceId) for traceId in traceIds))
            result = connection.execute(query)

            for row in result:
                trace = {}

                trace['serviceName'] = serviceName
                trace['spanCount'] = row['spanCount']
                trace['trace_id'] = row['trace_id']

                startTime = (int(row['created_ts']) / 1000000)
                trace['startTime'] = time.strftime('%m-%d-%YT%H:%M:%S', time.gmtime(startTime))

                servicesQuery = "SELECT service_name, `value`, a_timestamp \
                        FROM zipkin_annotations \
                        WHERE trace_id = %s AND \
                        `value` IN ('cs', 'sr', 'ss', 'cr') \
                        ORDER BY service_name ASC" % (row['trace_id'])
                servicesResult = connection.execute(servicesQuery)

                services = {}
                service = None
                for serviceRow in servicesResult:
                    if serviceRow['service_name'] not in services:
                        services[serviceRow['service_name']] = {}
                        service = services[serviceRow['service_name']]

                    service[serviceRow['value']] = serviceRow['a_timestamp']

                duration = 0
                serviceDuration = 0
                serviceDurations = []

                for key in services:
                    service = services[key]
                    if 'cs' in service:
                        serviceDuration = service['cr'] - service['cs']
                    else:
                        serviceDuration = service['ss'] - service['sr']

                    # service duration
                    serviceDurations.append({
                            'name': key,
                            'duration': serviceDuration
                        })

                    # adding up duration to get total duration time
                    duration = duration + serviceDuration

                # total duration for a trace
                trace['duration'] = duration

                # service durations

                # sort service durations
                serviceDurations = sorted(serviceDurations, key=lambda x: x['name'])

                trace['serviceDurations'] = serviceDurations

                traceResults.append( trace )

        #return json.dumps(traceResults)

    # populate services
    services = []
    result = connection.execute("SELECT DISTINCT service_name FROM zipkin_annotations")

    for row in result:
        services.append( row['service_name'] )

    spans = []

    if serviceName:
        query = "SELECT DISTINCT span_name FROM zipkin_annotations WHERE service_name='%s'" % serviceName
        result = connection.execute(query)

        for row in result:
            spans.append( row['span_name'] )

    if len(spans) > 0:
        spans.insert(0, 'all')

    # close connection
    connection.close()

    return render_template('index.html', \
            results=traceResults, \
            services=services, spans=spans, \
            get_SpanName=spanName, get_ServiceName=serviceName, \
            get_Timestamp=timestamp, get_Limit=limit)
<|endoftext|>


<commit_before>#!/usr/bin/env python

import sys, re, unittest as UT, xml.etree.ElementTree as ET
from os import path, getcwd
from errno import EISDIR, EINVAL, EACCES
from StringIO import StringIO
from subprocess import Popen, PIPE
from urllib2 import urlopen, HTTPError
from urlparse import urljoin, urlunparse
from traceback import format_exception

debug = False

# program to run - None means we still need to search for it
zbarimg = None
# arguments to said program
zbarimg_args = [ '-q', '--xml' ]

# namespace support
try:
    register_namespace = ET.register_namespace
except AttributeError:
    def register_namespace(prefix, uri):
        ET._namespace_map[uri] = prefix

# barcode results
BC = 'http://zbar.sourceforge.net/2008/barcode'
register_namespace('bc', BC)

# testcase extensions
TS = 'http://zbar.sourceforge.net/2009/test-spec'
register_namespace('test', TS)

# printing support
def fixtag(node):
    return str(node.tag).split('}', 1)[-1]

def toxml(node):
    s = StringIO()
    ET.ElementTree(node).write(s)
    return s.getvalue()

def hexdump(data):
    print data
    for i, c in enumerate(data):
        if i & 0x7 == 0:
            print '[%04x]' % i,
        print ' %04x' % ord(c),
        if i & 0x7 == 0x7:
            print
    if len(c) & 0x7 != 0x7:
        print '\n'

# search for a file in the distribution
def distdir_search(subdir, base, suffixes=('',)):
    # start with current dir,
    # then try script invocation path
    rundir = path.dirname(sys.argv[0])
    search = [ '', rundir ]

    # finally, attempt to follow VPATH if present
    try:
        import re
        with open('Makefile') as makefile:
            for line in makefile:
                if re.match('^VPATH\s*=', line):
                    vpath = line.split('=', 1)[1].strip()
                    if vpath and vpath != rundir:
                        search.append(vpath)
                    break
    except:
        pass

    # poke around for subdir
    subdirs = tuple((subdir, path.join('..', subdir), '..', ''))

    for prefix in search:
        for subdir in subdirs:
            for suffix in suffixes:
                file = path.realpath(path.join(prefix, subdir, base + suffix))
                if path.isfile(file):
                    return(file)
    return None

def find_zbarimg():
    """search for zbarimg program to run.
    """
    global zbarimg
    # look in dist dir first
    zbarimg = distdir_search('zbarimg', 'zbarimg', ('', '.exe'))
    if not zbarimg:
        # fall back to PATH
        zbarimg = 'zbarimg'
        if debug:
            print 'using zbarimg from PATH'
    elif debug:
        print 'using:', zbarimg

def run_zbarimg(images):
    """invoke zbarimg for the specified files.

    return results as an ET.Element
    """
    args = [ zbarimg ]
    args.extend(zbarimg_args)
    args.extend(images)
    if debug:
        print 'running:', ' '.join(args)

    # FIXME should be able to pipe (feed) parser straight from output
    child = Popen(args, stdout = PIPE, stderr = PIPE)
    (xml, err) = child.communicate()

    rc = child.returncode
    if debug:
        print 'zbarimg returned', rc

    # FIXME trim usage from error msg
    assert rc in (0, 4), \
           'zbarimg returned error status (%d)\n' % rc + err

    result = ET.XML(xml)
    assert result.tag == ET.QName(BC, 'barcodes')
    return result

class TestCase(UT.TestCase):
    """single barcode source test.

    must have source attribute set to an ET.Element representation of a
    bc:source tag before test is run.
    """
    def shortDescription(self):
        return self.source.get('href')

    def setUp(self):
        if not zbarimg:
            find_zbarimg()

    def runTest(self):
        expect = self.source
        assert expect is not None
        assert expect.tag == ET.QName(BC, 'source')
        actual = run_zbarimg((expect.get('href'),))

        self.assertEqual(len(actual), 1)

        try:
            compare_sources(expect, actual[0])
        except AssertionError:
            if expect.get(str(ET.QName(TS, 'exception'))) != 'TODO':
                raise
            # ignore

class BuiltinTestCase(TestCase):
    def __init__(self, methodName='runTest'):
        TestCase.__init__(self, methodName)

        href = distdir_search('examples', 'barcode.png')
        if not href:
            href = 'http://zbar.sf.net/test/barcode.png'

        self.source = src = ET.Element(ET.QName(BC, 'source'), href=href)
        sym = ET.SubElement(src, ET.QName(BC, 'symbol'), type='EAN-13')
        data = ET.SubElement(sym, ET.QName(BC, 'data'))
        data.text = '9876543210128'

def compare_maps(expect, actual, compare_func):
    errors = []
    notes = []
    for key, iact in actual.iteritems():
        iexp = expect.pop(key, None)
        if iexp is None:
            errors.append('bonus unexpected result:\n' + toxml(iact))
            continue

        try:
            compare_func(iexp, iact)
        except:
            errors.append(''.join(format_exception(*sys.exc_info())))

        if iexp.get(str(ET.QName(TS, 'exception'))) == 'TODO':
            notes.append('TODO unexpected result:\n' + toxml(iact))

    for key, iexp in expect.iteritems():
        if iexp.get(str(ET.QName(TS, 'exception'))) == 'TODO':
            notes.append('TODO missing expected result:\n' + toxml(iexp))
        else:
            errors.append('missing expected result:\n' + toxml(iexp))

    if len(notes) == 1:
        print >>sys.stderr, '(TODO)',
    elif notes:
        print >>sys.stderr, '(%d TODO)' % len(notes),
    assert not errors, '\n'.join(errors)

def compare_sources(expect, actual):
    assert actual.tag == ET.QName(BC, 'source')
    assert actual.get('href').endswith(expect.get('href')), \
           'source href mismatch: %s != %s' % (acthref, exphref)

    # FIXME process/trim test:* contents

    def map_source(src):
        if not len(src) or src[0].tag != ET.QName(BC, 'index'):
            # insert artificial hierarchy
            syms = src[:]
            del src[:]
            idx = ET.SubElement(src, ET.QName(BC, 'index'), num='0')
            idx[:] = syms
            exc = src.get(str(ET.QName(TS, 'exception')))
            if exc is not None:
                idx.set(str(ET.QName(TS, 'exception')), exc)
            return { '0': idx }
        elif len(src):
            assert src[0].tag != ET.QName(BC, 'symbol'), \
                   'invalid source element: ' + \
                   'expecting "index" or "symbol", got "%s"' % fixtag(src[0])

        srcmap = { }
        for idx in src:
            srcmap[idx.get('num')] = idx
        return srcmap

    compare_maps(map_source(expect), map_source(actual), compare_indices)

def compare_indices(expect, actual):
    assert actual.tag == ET.QName(BC, 'index')
    assert actual.get('num') == expect.get('num')

    # FIXME process/trim test:* contents

    def map_index(idx):
        idxmap = { }
        for sym in idx:
            assert sym.tag == ET.QName(BC, 'symbol')
            typ = sym.get('type')
            assert typ is not None
            data = sym.find(str(ET.QName(BC, 'data'))).text
            idxmap[typ, data] = sym

        return idxmap

    try:
        compare_maps(map_index(expect), map_index(actual), compare_symbols)
    except AssertionError:
        if expect.get(str(ET.QName(TS, 'exception'))) != 'TODO':
            raise

def compare_symbols(expect, actual):
    pass

# override unittest.TestLoader to populate tests from xml description
class TestLoader:
    suiteClass = UT.TestSuite

    def __init__(self):
        self.cwd = urlunparse(('file', '', getcwd() + '/', '', '', ''))
        if debug:
            print 'cwd =', self.cwd

    def loadTestsFromModule(self, module):
        return self.suiteClass([BuiltinTestCase()])

    def loadTestsFromNames(self, names, module=None):
        suites = [ self.loadTestsFromName(name, module) for name in names ]
        return self.suiteClass(suites)

    def loadTestsFromURL(self, url=None, file=None):
        if debug:
            print 'loading url:', url

        target = None
        if not file:
            if not url:
                return self.suiteClass([BuiltinTestCase()])

            content = None
            url = urljoin(self.cwd, url)
            # FIXME grok fragment

            try:
                if debug:
                    print 'trying:', url
                file = urlopen(url)
                content = file.info().get('Content-Type')
            except HTTPError, e:
                # possible remote directory
                pass
            except IOError, e:
                if e.errno not in (EISDIR, EINVAL, EACCES):
                    raise
                # could be local directory

            if (not file or
                content == 'text/html' or
                (isinstance(file, HTTPError) and file.code != 200)):
                # could be directory, try opening index
                try:
                    tmp = urljoin(url + '/', 'index.xml')
                    if debug:
                        print 'trying index:', tmp
                    file = urlopen(tmp)
                    content = file.info().get('Content-Type')
                    url = tmp
                except IOError:
                    raise IOError('no test index found at: %s' % url)

            if debug:
                print '\tContent-Type:', content

            if content not in ('application/xml', 'text/xml'):
                # assume url is image to test, try containing index
                # FIXME should be able to keep searching up
                try:
                    target = url.rsplit('/', 1)[1]
                    index = urljoin(url, 'index.xml')
                    if debug:
                        print 'trying index:', index
                    file = urlopen(index)
                    content = file.info().get('Content-Type')
                    if debug:
                        print '\tContent-Type:', content
                    assert content in ('application/xml', 'text/xml')
                    url = index
                except IOError:
                    raise IOError('no index found for: %s' % url)

        index = ET.ElementTree(file=file).getroot()
        assert index.tag == ET.QName(BC, 'barcodes')

        suite = self.suiteClass()
        for src in index:
            # FIXME trim any meta info
            href = src.get('href')
            if target and target != href:
                continue
            if src.tag == ET.QName(BC, 'source'):
                test = TestCase()
                src.set('href', urljoin(url, href))
                test.source = src
                suite.addTest(test)
            elif src.tag == ET.QName(TS, 'index'):
                suite.addTest(self.loadTestsFromURL(urljoin(url, href)))
            else:
                raise AssertionError('malformed test index') # FIXME detail

        assert suite.countTestCases(), 'empty test index: %s' % url
        return suite

    def loadTestsFromName(self, name=None, module=None):
        return self.loadTestsFromURL(name)

    def unsupported(self, *args, **kwargs):
        raise TypeError("unsupported TestLoader API")

    loadTestsFromTestCase = unsupported
    getTestCaseNames = unsupported

if __name__ == '__main__':
    if '-d' in sys.argv:
        debug = True
        sys.argv.remove('-d')

    UT.main(module=None, testLoader=TestLoader())
<commit_msg>more python test compatibility updates<commit_after>#!/usr/bin/env python

import sys, re, unittest as UT, xml.etree.ElementTree as ET
from os import path, getcwd
from errno import EISDIR, EINVAL, EACCES
from StringIO import StringIO
from subprocess import Popen, PIPE
from urllib2 import urlopen, HTTPError
from urlparse import urljoin, urlunparse
from traceback import format_exception

debug = False

# program to run - None means we still need to search for it
zbarimg = None
# arguments to said program
zbarimg_args = [ '-q', '--xml' ]

# namespace support
try:
    register_namespace = ET.register_namespace
except AttributeError:
    def register_namespace(prefix, uri):
        ET._namespace_map[uri] = prefix

# barcode results
BC = 'http://zbar.sourceforge.net/2008/barcode'
register_namespace('bc', BC)

# testcase extensions
TS = 'http://zbar.sourceforge.net/2009/test-spec'
register_namespace('test', TS)

# printing support
def fixtag(node):
    return str(node.tag).split('}', 1)[-1]

def toxml(node):
    s = StringIO()
    ET.ElementTree(node).write(s)
    return s.getvalue()

def hexdump(data):
    print data
    for i, c in enumerate(data):
        if i & 0x7 == 0:
            print '[%04x]' % i,
        print ' %04x' % ord(c),
        if i & 0x7 == 0x7:
            print
    if len(c) & 0x7 != 0x7:
        print '\n'

# search for a file in the distribution
def distdir_search(subdir, base, suffixes=('',)):
    # start with current dir,
    # then try script invocation path
    rundir = path.dirname(sys.argv[0])
    search = [ '', rundir ]

    # finally, attempt to follow VPATH if present
    try:
        import re
        makefile = open('Makefile')
        for line in makefile:
            if re.match('^VPATH\s*=', line):
                vpath = line.split('=', 1)[1].strip()
                if vpath and vpath != rundir:
                    search.append(vpath)
                break
    except:
        pass

    # poke around for subdir
    subdirs = tuple((subdir, path.join('..', subdir), '..', ''))

    for prefix in search:
        for subdir in subdirs:
            for suffix in suffixes:
                file = path.realpath(path.join(prefix, subdir, base + suffix))
                if path.isfile(file):
                    return(file)
    return None

def find_zbarimg():
    """search for zbarimg program to run.
    """
    global zbarimg
    # look in dist dir first
    zbarimg = distdir_search('zbarimg', 'zbarimg', ('', '.exe'))
    if not zbarimg:
        # fall back to PATH
        zbarimg = 'zbarimg'
        if debug:
            print 'using zbarimg from PATH'
    elif debug:
        print 'using:', zbarimg

def run_zbarimg(images):
    """invoke zbarimg for the specified files.

    return results as an ET.Element
    """
    args = [ zbarimg ]
    args.extend(zbarimg_args)
    args.extend(images)
    if debug:
        print 'running:', ' '.join(args)

    # FIXME should be able to pipe (feed) parser straight from output
    child = Popen(args, stdout = PIPE, stderr = PIPE)
    (xml, err) = child.communicate()

    rc = child.returncode
    if debug:
        print 'zbarimg returned', rc

    # FIXME trim usage from error msg
    assert rc in (0, 4), \
           'zbarimg returned error status (%d)\n' % rc + err

    result = ET.XML(xml)
    assert result.tag == ET.QName(BC, 'barcodes')
    return result

class TestCase(UT.TestCase):
    """single barcode source test.

    must have source attribute set to an ET.Element representation of a
    bc:source tag before test is run.
    """
    def shortDescription(self):
        return self.source.get('href')

    def setUp(self):
        if not zbarimg:
            find_zbarimg()

    def runTest(self):
        expect = self.source
        assert expect is not None
        assert expect.tag == ET.QName(BC, 'source')
        actual = run_zbarimg((expect.get('href'),))

        self.assertEqual(len(actual), 1)

        try:
            compare_sources(expect, actual[0])
        except AssertionError:
            if expect.get(str(ET.QName(TS, 'exception'))) != 'TODO':
                raise
            # ignore

class BuiltinTestCase(TestCase):
    def __init__(self, methodName='runTest'):
        TestCase.__init__(self, methodName)

        href = distdir_search('examples', 'barcode.png')
        if not href:
            href = 'http://zbar.sf.net/test/barcode.png'

        self.source = src = ET.Element(ET.QName(BC, 'source'), href=href)
        sym = ET.SubElement(src, ET.QName(BC, 'symbol'), type='EAN-13')
        data = ET.SubElement(sym, ET.QName(BC, 'data'))
        data.text = '9876543210128'

def compare_maps(expect, actual, compare_func):
    errors = []
    notes = []
    for key, iact in actual.iteritems():
        iexp = expect.pop(key, None)
        if iexp is None:
            errors.append('bonus unexpected result:\n' + toxml(iact))
            continue

        try:
            compare_func(iexp, iact)
        except:
            errors.append(''.join(format_exception(*sys.exc_info())))

        if iexp.get(str(ET.QName(TS, 'exception'))) == 'TODO':
            notes.append('TODO unexpected result:\n' + toxml(iact))

    for key, iexp in expect.iteritems():
        if iexp.get(str(ET.QName(TS, 'exception'))) == 'TODO':
            notes.append('TODO missing expected result:\n' + toxml(iexp))
        else:
            errors.append('missing expected result:\n' + toxml(iexp))

    if len(notes) == 1:
        print >>sys.stderr, '(TODO)',
    elif notes:
        print >>sys.stderr, '(%d TODO)' % len(notes),
    assert not errors, '\n'.join(errors)

def compare_sources(expect, actual):
    assert actual.tag == ET.QName(BC, 'source')
    assert actual.get('href').endswith(expect.get('href')), \
           'source href mismatch: %s != %s' % (acthref, exphref)

    # FIXME process/trim test:* contents

    def map_source(src):
        if not len(src) or src[0].tag != ET.QName(BC, 'index'):
            # insert artificial hierarchy
            syms = src[:]
            del src[:]
            idx = ET.SubElement(src, ET.QName(BC, 'index'), num='0')
            idx[:] = syms
            exc = src.get(str(ET.QName(TS, 'exception')))
            if exc is not None:
                idx.set(str(ET.QName(TS, 'exception')), exc)
            return { '0': idx }
        elif len(src):
            assert src[0].tag != ET.QName(BC, 'symbol'), \
                   'invalid source element: ' + \
                   'expecting "index" or "symbol", got "%s"' % fixtag(src[0])

        srcmap = { }
        for idx in src:
            srcmap[idx.get('num')] = idx
        return srcmap

    compare_maps(map_source(expect), map_source(actual), compare_indices)

def compare_indices(expect, actual):
    assert actual.tag == ET.QName(BC, 'index')
    assert actual.get('num') == expect.get('num')

    # FIXME process/trim test:* contents

    def map_index(idx):
        idxmap = { }
        for sym in idx:
            assert sym.tag == ET.QName(BC, 'symbol')
            typ = sym.get('type')
            assert typ is not None
            data = sym.find(str(ET.QName(BC, 'data'))).text
            idxmap[typ, data] = sym

        return idxmap

    try:
        compare_maps(map_index(expect), map_index(actual), compare_symbols)
    except AssertionError:
        if expect.get(str(ET.QName(TS, 'exception'))) != 'TODO':
            raise

def compare_symbols(expect, actual):
    pass

# override unittest.TestLoader to populate tests from xml description
class TestLoader:
    suiteClass = UT.TestSuite

    def __init__(self):
        self.cwd = urlunparse(('file', '', getcwd() + '/', '', '', ''))
        if debug:
            print 'cwd =', self.cwd

    def loadTestsFromModule(self, module):
        return self.suiteClass([BuiltinTestCase()])

    def loadTestsFromNames(self, names, module=None):
        suites = [ self.loadTestsFromName(name, module) for name in names ]
        return self.suiteClass(suites)

    def loadTestsFromURL(self, url=None, file=None):
        if debug:
            print 'loading url:', url

        target = None
        if not file:
            if not url:
                return self.suiteClass([BuiltinTestCase()])

            content = None
            url = urljoin(self.cwd, url)
            # FIXME grok fragment

            try:
                if debug:
                    print 'trying:', url
                file = urlopen(url)
                content = file.info().get('Content-Type')
            except HTTPError, e:
                # possible remote directory
                pass
            except IOError, e:
                if e.errno not in (EISDIR, EINVAL, EACCES):
                    raise
                # could be local directory

            if (not file or
                content == 'text/html' or
                (isinstance(file, HTTPError) and file.code != 200)):
                # could be directory, try opening index
                try:
                    tmp = urljoin(url + '/', 'index.xml')
                    if debug:
                        print 'trying index:', tmp
                    file = urlopen(tmp)
                    content = file.info().get('Content-Type')
                    url = tmp
                except IOError:
                    raise IOError('no test index found at: %s' % url)

            if debug:
                print '\tContent-Type:', content

            if content not in ('application/xml', 'text/xml'):
                # assume url is image to test, try containing index
                # FIXME should be able to keep searching up
                try:
                    target = url.rsplit('/', 1)[1]
                    index = urljoin(url, 'index.xml')
                    if debug:
                        print 'trying index:', index
                    file = urlopen(index)
                    content = file.info().get('Content-Type')
                    if debug:
                        print '\tContent-Type:', content
                    assert content in ('application/xml', 'text/xml')
                    url = index
                except IOError:
                    raise IOError('no index found for: %s' % url)

        index = ET.ElementTree(file=file).getroot()
        assert index.tag == ET.QName(BC, 'barcodes')

        suite = self.suiteClass()
        for src in index:
            # FIXME trim any meta info
            href = src.get('href')
            if target and target != href:
                continue
            if src.tag == ET.QName(BC, 'source'):
                test = TestCase()
                src.set('href', urljoin(url, href))
                test.source = src
                suite.addTest(test)
            elif src.tag == ET.QName(TS, 'index'):
                suite.addTest(self.loadTestsFromURL(urljoin(url, href)))
            else:
                raise AssertionError('malformed test index') # FIXME detail

        assert suite.countTestCases(), 'empty test index: %s' % url
        return suite

    def loadTestsFromName(self, name=None, module=None):
        return self.loadTestsFromURL(name)

    def unsupported(self, *args, **kwargs):
        raise TypeError("unsupported TestLoader API")

    loadTestsFromTestCase = unsupported
    getTestCaseNames = unsupported

if __name__ == '__main__':
    if '-d' in sys.argv:
        debug = True
        sys.argv.remove('-d')

    UT.main(module=None, testLoader=TestLoader())
<|endoftext|>


<commit_before>#####################################################################
# communication_log_file_handler.py
#
# (c) Copyright 2016, Benjamin Parzella. All rights reserved.
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2.1 of the License, or (at your option) any later version.
#
# This software is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Lesser General Public License for more details.
#####################################################################

import os
import logging

class CommunicationLogFileHandler(logging.Handler):
    def __init__(self, path, prefix=""):
        logging.Handler.__init__(self)

        self.path = path
        self.prefix = prefix

    def emit(self, record):
        filename = os.path.join(self.path, "{}com_{}.log".format(self.prefix, record.remoteName))
        os.makedirs(os.path.dirname(filename), exist_ok=True)
	with open(filename, 'a') as f:
            f.write(self.format(record) + "\n")
<commit_msg>Fix spaces/tabs<commit_after>#####################################################################
# communication_log_file_handler.py
#
# (c) Copyright 2016, Benjamin Parzella. All rights reserved.
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2.1 of the License, or (at your option) any later version.
#
# This software is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Lesser General Public License for more details.
#####################################################################

import os
import logging

class CommunicationLogFileHandler(logging.Handler):
    def __init__(self, path, prefix=""):
        logging.Handler.__init__(self)

        self.path = path
        self.prefix = prefix

    def emit(self, record):
        filename = os.path.join(self.path, "{}com_{}.log".format(self.prefix, record.remoteName))
        os.makedirs(os.path.dirname(filename), exist_ok=True)
        with open(filename, 'a') as f:
            f.write(self.format(record) + "\n")
<|endoftext|>