<commit_before># Copyright 2020 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Provides a workspace macro to load dependent repositories."""

load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
load("@bazel_tools//tools/build_defs/repo:utils.bzl", "maybe")
load("@tf_runtime//third_party:repo.bzl", "tfrt_http_archive")
load("@tf_runtime//third_party/cuda:dependencies.bzl", "cuda_dependencies")

def tfrt_dependencies():
    """Loads TFRT external dependencies into WORKSPACE."""

    # Bazel rules to build CUDA targets.
    # TODO(csigg): move this to some public repo.
    maybe(
        name = "rules_cuda",
        repo_rule = native.local_repository,
        path = "third_party/rules_cuda",
    )

    cuda_dependencies()

    # LLVM and MLIR from llvm-project.
    LLVM_COMMIT = "1426ac0482951d4cb98f84ab9fc89c745f95602e"
    LLVM_SHA256 = "51d7b55551523336dfbb122f5296a931d6053937c69a71d61749b1e6af53b153"
    LLVM_URLS = [
        "https://storage.googleapis.com/mirror.tensorflow.org/github.com/llvm/llvm-project/archive/{commit}.tar.gz".format(commit = LLVM_COMMIT),
        "https://github.com/llvm/llvm-project/archive/{commit}.tar.gz".format(commit = LLVM_COMMIT),
    ]
    maybe(
        name = "llvm-project",
        repo_rule = tfrt_http_archive,
        sha256 = LLVM_SHA256,
        strip_prefix = "llvm-project-" + LLVM_COMMIT,
        urls = LLVM_URLS,
        additional_build_files = {
            "@tf_runtime//third_party/llvm:llvm.autogenerated.BUILD": "llvm/BUILD",
            "@tf_runtime//third_party/mlir:BUILD": "mlir/BUILD",
            "@tf_runtime//third_party/mlir:test.BUILD": "mlir/test/BUILD",
        },
    )

    # https://github.com/bazelbuild/bazel-skylib/releases
    maybe(
        name = "bazel_skylib",
        repo_rule = http_archive,
        sha256 = "97e70364e9249702246c0e9444bccdc4b847bed1eb03c5a3ece4f83dfe6abc44",
        urls = [
            "https://storage.googleapis.com/mirror.tensorflow.org/github.com/bazelbuild/bazel-skylib/releases/download/1.0.2/bazel-skylib-1.0.2.tar.gz",
            "https://github.com/bazelbuild/bazel-skylib/releases/download/1.0.2/bazel-skylib-1.0.2.tar.gz",
        ],
    )

    # Eigen3
    maybe(
        name = "eigen_archive",
        repo_rule = http_archive,
        build_file = "@tf_runtime//third_party:eigen/BUILD",
        sha256 = "924c7f85d5e2e40beb663489bf3f6908bb4c328dc81ab845c27bb6ce199b8698",  # SHARED_EIGEN_SHA
        strip_prefix = "eigen-3c02fefec53f21d9fad537ff0d62d8dcc8162466",
        urls = [
            "https://storage.googleapis.com/mirror.tensorflow.org/gitlab.com/libeigen/eigen/-/archive/3c02fefec53f21d9fad537ff0d62d8dcc8162466/eigen-3c02fefec53f21d9fad537ff0d62d8dcc8162466.tar.gz",
            "https://gitlab.com/libeigen/eigen/-/archive/3c02fefec53f21d9fad537ff0d62d8dcc8162466/eigen-3c02fefec53f21d9fad537ff0d62d8dcc8162466.tar.gz",
        ],
    )

    maybe(
        name = "mkl_dnn",
        repo_rule = http_archive,
        build_file = "@tf_runtime//third_party/mkl_dnn:BUILD.mkl_dnn",
        sha256 = "ed56652dd237deb86ee9bf102c18de5f2625c059e5ab1d7512c8dc01e316b694",
        strip_prefix = "mkl-dnn-0.21.2",
        urls = [
            "https://storage.googleapis.com/mirror.tensorflow.org/github.com/intel/mkl-dnn/archive/v0.21.2.tar.gz",
            "https://github.com/intel/mkl-dnn/archive/v0.21.2.tar.gz",
        ],
    )

    maybe(
        name = "com_google_googletest",
        repo_rule = tfrt_http_archive,
        sha256 = "ff7a82736e158c077e76188232eac77913a15dac0b22508c390ab3f88e6d6d86",
        strip_prefix = "googletest-b6cd405286ed8635ece71c72f118e659f4ade3fb",
        urls = [
            "https://storage.googleapis.com/mirror.tensorflow.org/github.com/google/googletest/archive/b6cd405286ed8635ece71c72f118e659f4ade3fb.zip",
            "https://github.com/google/googletest/archive/b6cd405286ed8635ece71c72f118e659f4ade3fb.zip",
        ],
    )

    maybe(
        name = "com_github_google_benchmark",
        repo_rule = http_archive,
        strip_prefix = "benchmark-16703ff83c1ae6d53e5155df3bb3ab0bc96083be",
        sha256 = "59f918c8ccd4d74b6ac43484467b500f1d64b40cc1010daa055375b322a43ba3",
        urls = ["https://github.com/google/benchmark/archive/16703ff83c1ae6d53e5155df3bb3ab0bc96083be.zip"],
    )

    # Protobuf
    PROTOBUF_URLS = [
        # 310ba5ee72661c081129eb878c1bbcec936b20f0 is based on 3.8.0 with a fix
        # for protobuf.bzl.
        "https://storage.googleapis.com/mirror.tensorflow.org/github.com/protocolbuffers/protobuf/archive/310ba5ee72661c081129eb878c1bbcec936b20f0.tar.gz",
        "https://github.com/protocolbuffers/protobuf/archive/310ba5ee72661c081129eb878c1bbcec936b20f0.tar.gz",
    ]
    PROTOBUF_SHA256 = "b9e92f9af8819bbbc514e2902aec860415b70209f31dfc8c4fa72515a5df9d59"
    PROTOBUF_STRIP_PREFIX = "protobuf-310ba5ee72661c081129eb878c1bbcec936b20f0"
    PROTOBUF_PATCH = "//third_party/protobuf:protobuf.patch"
    maybe(
        name = "com_google_protobuf",
        repo_rule = tfrt_http_archive,
        patch_file = PROTOBUF_PATCH,
        sha256 = PROTOBUF_SHA256,
        strip_prefix = PROTOBUF_STRIP_PREFIX,
        system_build_file = "//third_party/systemlibs:protobuf.BUILD",
        system_link_files = {
            "//third_party/systemlibs:protobuf.bzl": "protobuf.bzl",
        },
        urls = PROTOBUF_URLS,
    )

    maybe(
        name = "cub_archive",
        repo_rule = tfrt_http_archive,
        build_file = "//third_party:cub/BUILD",
        patch_file = "//third_party:cub/pr170.patch",
        sha256 = "6bfa06ab52a650ae7ee6963143a0bbc667d6504822cbd9670369b598f18c58c3",
        strip_prefix = "cub-1.8.0",
        urls = [
            "https://storage.googleapis.com/mirror.tensorflow.org/github.com/NVlabs/cub/archive/1.8.0.zip",
            "https://github.com/NVlabs/cub/archive/1.8.0.zip",
        ],
    )

    maybe(
        name = "zlib",
        repo_rule = tfrt_http_archive,
        build_file = "//third_party:zlib.BUILD",
        sha256 = "c3e5e9fdd5004dcb542feda5ee4f0ff0744628baf8ed2dd5d66f8ca1197cb1a1",
        strip_prefix = "zlib-1.2.11",
        system_build_file = "//third_party/systemlibs:zlib.BUILD",
        urls = [
            "https://storage.googleapis.com/mirror.tensorflow.org/zlib.net/zlib-1.2.11.tar.gz",
            "https://zlib.net/zlib-1.2.11.tar.gz",
        ],
    )

    maybe(
        name = "py-cpuinfo",
        repo_rule = tfrt_http_archive,
        strip_prefix = "py-cpuinfo-0.2.3",
        sha256 = "f6a016fdbc4e7fadf2d519090fcb4fa9d0831bad4e85245d938e5c2fe7623ca6",
        urls = [
            "https://storage.googleapis.com/mirror.tensorflow.org/pypi.python.org/packages/source/p/py-cpuinfo/py-cpuinfo-0.2.3.tar.gz",
            "https://pypi.python.org/packages/source/p/py-cpuinfo/py-cpuinfo-0.2.3.tar.gz",
        ],
        build_file = "//third_party:py-cpuinfo.BUILD",
    )
<commit_msg>Integrate LLVM at llvm/llvm-project@dc8d7d23d8d2<commit_after># Copyright 2020 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Provides a workspace macro to load dependent repositories."""

load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
load("@bazel_tools//tools/build_defs/repo:utils.bzl", "maybe")
load("@tf_runtime//third_party:repo.bzl", "tfrt_http_archive")
load("@tf_runtime//third_party/cuda:dependencies.bzl", "cuda_dependencies")

def tfrt_dependencies():
    """Loads TFRT external dependencies into WORKSPACE."""

    # Bazel rules to build CUDA targets.
    # TODO(csigg): move this to some public repo.
    maybe(
        name = "rules_cuda",
        repo_rule = native.local_repository,
        path = "third_party/rules_cuda",
    )

    cuda_dependencies()

    # LLVM and MLIR from llvm-project.
    LLVM_COMMIT = "dc8d7d23d8d25e1ba4859c831151158ae6a617ef"
    LLVM_SHA256 = "fe005ab69c846cd4eacb8c7ab479c7558ccf0edbd8e404c12d47427d30c7fc69"
    LLVM_URLS = [
        "https://storage.googleapis.com/mirror.tensorflow.org/github.com/llvm/llvm-project/archive/{commit}.tar.gz".format(commit = LLVM_COMMIT),
        "https://github.com/llvm/llvm-project/archive/{commit}.tar.gz".format(commit = LLVM_COMMIT),
    ]
    maybe(
        name = "llvm-project",
        repo_rule = tfrt_http_archive,
        sha256 = LLVM_SHA256,
        strip_prefix = "llvm-project-" + LLVM_COMMIT,
        urls = LLVM_URLS,
        additional_build_files = {
            "@tf_runtime//third_party/llvm:llvm.autogenerated.BUILD": "llvm/BUILD",
            "@tf_runtime//third_party/mlir:BUILD": "mlir/BUILD",
            "@tf_runtime//third_party/mlir:test.BUILD": "mlir/test/BUILD",
        },
    )

    # https://github.com/bazelbuild/bazel-skylib/releases
    maybe(
        name = "bazel_skylib",
        repo_rule = http_archive,
        sha256 = "97e70364e9249702246c0e9444bccdc4b847bed1eb03c5a3ece4f83dfe6abc44",
        urls = [
            "https://storage.googleapis.com/mirror.tensorflow.org/github.com/bazelbuild/bazel-skylib/releases/download/1.0.2/bazel-skylib-1.0.2.tar.gz",
            "https://github.com/bazelbuild/bazel-skylib/releases/download/1.0.2/bazel-skylib-1.0.2.tar.gz",
        ],
    )

    # Eigen3
    maybe(
        name = "eigen_archive",
        repo_rule = http_archive,
        build_file = "@tf_runtime//third_party:eigen/BUILD",
        sha256 = "924c7f85d5e2e40beb663489bf3f6908bb4c328dc81ab845c27bb6ce199b8698",  # SHARED_EIGEN_SHA
        strip_prefix = "eigen-3c02fefec53f21d9fad537ff0d62d8dcc8162466",
        urls = [
            "https://storage.googleapis.com/mirror.tensorflow.org/gitlab.com/libeigen/eigen/-/archive/3c02fefec53f21d9fad537ff0d62d8dcc8162466/eigen-3c02fefec53f21d9fad537ff0d62d8dcc8162466.tar.gz",
            "https://gitlab.com/libeigen/eigen/-/archive/3c02fefec53f21d9fad537ff0d62d8dcc8162466/eigen-3c02fefec53f21d9fad537ff0d62d8dcc8162466.tar.gz",
        ],
    )

    maybe(
        name = "mkl_dnn",
        repo_rule = http_archive,
        build_file = "@tf_runtime//third_party/mkl_dnn:BUILD.mkl_dnn",
        sha256 = "ed56652dd237deb86ee9bf102c18de5f2625c059e5ab1d7512c8dc01e316b694",
        strip_prefix = "mkl-dnn-0.21.2",
        urls = [
            "https://storage.googleapis.com/mirror.tensorflow.org/github.com/intel/mkl-dnn/archive/v0.21.2.tar.gz",
            "https://github.com/intel/mkl-dnn/archive/v0.21.2.tar.gz",
        ],
    )

    maybe(
        name = "com_google_googletest",
        repo_rule = tfrt_http_archive,
        sha256 = "ff7a82736e158c077e76188232eac77913a15dac0b22508c390ab3f88e6d6d86",
        strip_prefix = "googletest-b6cd405286ed8635ece71c72f118e659f4ade3fb",
        urls = [
            "https://storage.googleapis.com/mirror.tensorflow.org/github.com/google/googletest/archive/b6cd405286ed8635ece71c72f118e659f4ade3fb.zip",
            "https://github.com/google/googletest/archive/b6cd405286ed8635ece71c72f118e659f4ade3fb.zip",
        ],
    )

    maybe(
        name = "com_github_google_benchmark",
        repo_rule = http_archive,
        strip_prefix = "benchmark-16703ff83c1ae6d53e5155df3bb3ab0bc96083be",
        sha256 = "59f918c8ccd4d74b6ac43484467b500f1d64b40cc1010daa055375b322a43ba3",
        urls = ["https://github.com/google/benchmark/archive/16703ff83c1ae6d53e5155df3bb3ab0bc96083be.zip"],
    )

    # Protobuf
    PROTOBUF_URLS = [
        # 310ba5ee72661c081129eb878c1bbcec936b20f0 is based on 3.8.0 with a fix
        # for protobuf.bzl.
        "https://storage.googleapis.com/mirror.tensorflow.org/github.com/protocolbuffers/protobuf/archive/310ba5ee72661c081129eb878c1bbcec936b20f0.tar.gz",
        "https://github.com/protocolbuffers/protobuf/archive/310ba5ee72661c081129eb878c1bbcec936b20f0.tar.gz",
    ]
    PROTOBUF_SHA256 = "b9e92f9af8819bbbc514e2902aec860415b70209f31dfc8c4fa72515a5df9d59"
    PROTOBUF_STRIP_PREFIX = "protobuf-310ba5ee72661c081129eb878c1bbcec936b20f0"
    PROTOBUF_PATCH = "//third_party/protobuf:protobuf.patch"
    maybe(
        name = "com_google_protobuf",
        repo_rule = tfrt_http_archive,
        patch_file = PROTOBUF_PATCH,
        sha256 = PROTOBUF_SHA256,
        strip_prefix = PROTOBUF_STRIP_PREFIX,
        system_build_file = "//third_party/systemlibs:protobuf.BUILD",
        system_link_files = {
            "//third_party/systemlibs:protobuf.bzl": "protobuf.bzl",
        },
        urls = PROTOBUF_URLS,
    )

    maybe(
        name = "cub_archive",
        repo_rule = tfrt_http_archive,
        build_file = "//third_party:cub/BUILD",
        patch_file = "//third_party:cub/pr170.patch",
        sha256 = "6bfa06ab52a650ae7ee6963143a0bbc667d6504822cbd9670369b598f18c58c3",
        strip_prefix = "cub-1.8.0",
        urls = [
            "https://storage.googleapis.com/mirror.tensorflow.org/github.com/NVlabs/cub/archive/1.8.0.zip",
            "https://github.com/NVlabs/cub/archive/1.8.0.zip",
        ],
    )

    maybe(
        name = "zlib",
        repo_rule = tfrt_http_archive,
        build_file = "//third_party:zlib.BUILD",
        sha256 = "c3e5e9fdd5004dcb542feda5ee4f0ff0744628baf8ed2dd5d66f8ca1197cb1a1",
        strip_prefix = "zlib-1.2.11",
        system_build_file = "//third_party/systemlibs:zlib.BUILD",
        urls = [
            "https://storage.googleapis.com/mirror.tensorflow.org/zlib.net/zlib-1.2.11.tar.gz",
            "https://zlib.net/zlib-1.2.11.tar.gz",
        ],
    )

    maybe(
        name = "py-cpuinfo",
        repo_rule = tfrt_http_archive,
        strip_prefix = "py-cpuinfo-0.2.3",
        sha256 = "f6a016fdbc4e7fadf2d519090fcb4fa9d0831bad4e85245d938e5c2fe7623ca6",
        urls = [
            "https://storage.googleapis.com/mirror.tensorflow.org/pypi.python.org/packages/source/p/py-cpuinfo/py-cpuinfo-0.2.3.tar.gz",
            "https://pypi.python.org/packages/source/p/py-cpuinfo/py-cpuinfo-0.2.3.tar.gz",
        ],
        build_file = "//third_party:py-cpuinfo.BUILD",
    )
<|endoftext|>


<commit_before>#!/usr/bin/python
# pylint: disable=line-too-long
"""
module for TiemseriesArray Class
"""
import logging
import base64
import json
import os
import gzip
import numpy
# own modules
from Timeseries import Timeseries as Timeseries

def is_near(value, target_value, pct=0.05):
    """
    function to implement if some numeric is near, in terms of percent
    handy to use with floating point equality
    """
    minimum = float(target_value) * (1.0 - pct)
    maximum = float(target_value) * (1.0 + pct)
    return minimum < float(value) < maximum

class TimeseriesArray(object):
    """
    holds dictionary of Timeseries objects
    """
    group_funcs = {
        "sum" : lambda a: sum(a),
        "min" : lambda a: min(a),
        "max" : lambda a: max(a),
        "avg" : lambda a: sum(a) / len(a),
        "len" : lambda a: len(a),
    }

    def __init__(self, index_keys, value_keys, ts_key="ts"):
        """
        index_keys <tuple> column names of index columns
        value_keys <tuple> column names of value columns
        ts_key <str> name of timestamp column
        """
        self.__index_keys = tuple([unicode(value) for value in index_keys])
        self.__value_keys = list([unicode(value) for value in value_keys])
        self.__ts_key = unicode(ts_key)
        # define instance data
        self.__data = {} # holds data

    def __len__(self):
        """mimic dict"""
        return len(self.__data.keys())

    def __getitem__(self, key):
        """mimic dict"""
        return self.__data[key]

    def __setitem__(self, key, value):
        """mimic dict"""
        self.__data[key] = value

    def __delitem__(self, key):
        """mimic dict"""
        del self.__data[key]

    def __eq__(self, other):
        """test equality in depth"""
        try:
            assert self.__index_keys == other.index_keys
            assert self.__value_keys == other.value_keys
            assert self.__ts_key == other.ts_key
            assert len(self.__data.keys()) == len(other.keys())
            for key in self.__data.keys():
                if len(self.__data[key]) != len(other.data[key]):
                    raise AssertionError("timeseries %s data differences in length" % key)
            return True
        except AssertionError as exc:
            logging.exception(exc)
            return False

    def items(self):
        """mimic dict"""
        return self.__data.items()

    def values(self):
        """mimic dict"""
        return self.__data.values()

    def keys(self):
        """
        return keys of internal data object
        mimic dict behaviour
        """
        return self.__data.keys()

    def get_index_dict(self, index_values):
        """
        index_values <tuple> as used in self.data.keys()

        return <dict> representation of given index_values
        """
        return dict(zip(self.__index_keys, index_values))

    @property
    def data(self):
        return self.__data

    @data.setter
    def data(self, value):
        raise NotImplementedError("setting of data is not allowed, use add() or append()")
        self.__data = value

    @property
    def index_keys(self):
        return self.__index_keys

    @index_keys.setter
    def index_keys(self, value):
        raise NotImplementedError("setting of index_keys is not allowed")
        self.__index_keys = tuple(value)

#    def get_index_keys(self):
#        return tuple(self.index_keys)

    @property
    def value_keys(self):
        return self.__value_keys

    @value_keys.setter
    def value_keys(self, value):
        raise NotImplementedError("setting of value_keys is not allowed")
        self.__value_keys = list(value)

#    def get_value_keys(self):
#        return self.value_keys

    @property
    def ts_key(self):
        return self.__ts_key

    @ts_key.setter
    def ts_key(self, value):
        raise NotImplementedError("setting of ts_key is not allowed")
        self.__ts_key = value

#    def get_ts_key(self):
#        return self.ts_key

#    def get_first_ts(self):
#        first_ts = min(series[0][0] for series in self.data.values())
#        #logging.info("first ts found in data : %s", first_ts)
#        return first_ts

#    def get_last_ts(self):
#        last_ts = max(series[-1][0] for series in self.data.values())
#        #logging.info("first ts found in data : %s", last_ts)
#        return last_ts

    def add(self, data):
        """
        data must have following keys
        <ts_keyname> <index_keys> <value_keys>
        if there are additional keys, these will be ignored

        All index_keys are converted to unicode
        All value_keys are converted to float
        ts_keyname is converted to float
        """
        #assert self.__ts_key in data # timestamp key has to be in dict
        #assert (type(data[self.__ts_key]) == int) or (type(data[self.__ts_key]) == float) # timestamp should be int
        #assert all((value_key in data for value_key in self.__value_keys)) # test if all keys are available
        # create key from data
        try:
            key = tuple([unicode(data[key]) for key in self.__index_keys])
        except KeyError:
            #logging.exception(exc)
            logging.error("there are index_keys missing in this dataset, skipping")
            return
        # if this key is new, create empty Timeseries object
        if key not in self.__data:
            self.__data[key] = Timeseries(self.__value_keys)
        # add data to this timeseries object
        try:
            # timestamp and values has to be converted to float
            self.__data[key].add(float(data[self.__ts_key]), tuple((float(data[key]) for key in self.__value_keys)))
        except KeyError as exc:
            #logging.exception(exc)
            logging.error("there is some key missing in %s, should be %s and %s, skipping this dataset, skipping this dataset", data.keys(), self.__ts_key, self.__value_keys)
        except ValueError as exc:
            #logging.exception(exc)
            logging.error("some value_keys or ts_keyname are not numeric and float convertible, skipping this dataset: %s", data)

    def append(self, key, timeserie):
        """
        key <str> key to store this timeseries in dict
        timeserie <Timeseries> object that holds the data

        append whole timeserie to existing data
        data length must be the same, but start_ts and stop_ts can be slightly different for each key
        """
        #logging.debug("new start : %s, stop: %s, length %s", timeserie[0][0], timeserie[-1][0], len(timeserie))
        assert key not in self.__data
        if len(self.__data) > 0:
            #logging.debug("existing start : %s, stop: %s, length %s", self.data.values()[0][0][0], self.data.values()[0][-1][0], len(self.data.values()[0]))
            try:
                assert len(timeserie) == len(self.__data.values()[0]) # must be the same length
            except AssertionError:
                logging.error("The Timeseries Object to append, has not the same length as existing data in this array")
                logging.error("existing: %d, new %d", len(timeserie), len(self.__data.values()[0]))
        else:
            logging.debug("this is the first timeseries")
        self.__data[key] = timeserie

    def groupby(self, fieldnum, group_func, time_func="avg"):
        """
        fieldnum <int>
        group_func <function> to group a set of numeric values
        time_func

        aggregate data on one of the index keys, example

        <ts1> <hostname> <intance0> <value1>
        <ts1> <hostname> <intance1> <value2>
        <ts1> <hostname> <intance2> <value3>
        <ts2> <hostname> <intance0> <value1> # here timeseries 2 starts
        <ts2> <hostname> <intance1> <value2>
        <ts2> <hostname> <intance2> <value3>

        to

        <hostname> group_func(time_func(value1) + time_func(value2) + time_func(value3))
        """
        ret_data = {}
        for key in self.__data:
            subkey = key[fieldnum]
            try:
                stat_data = self.__data[key].get_stat(time_func)
                if subkey not in ret_data:
                    ret_data[subkey] = stat_data
                    ret_data[subkey]["count"] = 1
                else:
                    #aggregate, there are more than one row who match
                    for valuekey in stat_data:
                        try:
                            ret_data[subkey][valuekey] = group_func(ret_data[subkey][valuekey], stat_data[valuekey])
                        except TypeError as exc:
                            logging.exception(exc)
                            logging.error("error at operation group_func(%s, %s)", ret_data[subkey][valuekey], stat_data[valuekey])
                    ret_data[subkey]["count"] += 1
            except StandardError as exc:
                logging.exception(exc)
        return ret_data

    def add_derive_col(self, colname, newcolname):
        """
        add one to key to every Timeseries, for this specific colname
        which represents the difference between two values in time
        TODO: describe this better
        """
        assert colname in self.__value_keys
        assert newcolname not in self.__value_keys
        for key, timeseries in self.__data.items():
            try:
                assert newcolname not in timeseries.headers
            except AssertionError as exc:
                logging.error("%s does exist in current dataset at key %s", newcolname, key)
                raise exc
            timeseries.add_derive_col(colname, newcolname)
        self.__value_keys.append(newcolname)

    def add_per_s_col(self, colname, newcolname):
        """
        add one to key to every Timeseries, for this specific colname
        which represents the difference between two values in time
        TODO: describe this better
        """
        assert colname in self.__value_keys
        assert newcolname not in self.__value_keys
        for timeseries in self.__data.values():
            assert newcolname not in timeseries.headers
            timeseries.add_per_s_col(colname, newcolname)
        self.__value_keys.append(newcolname)

    def add_calc_col_single(self, colname, newcolname, func=lambda a: a):
        """
        add a new column to every Timeserie, calculated from one single column in this row

        parameters:
        colname <str> name of existing column
        newcolname <str> name of new column
        func <func> function to call for every row, wtih data from colname
            lambda <float> : <float>

        return:
        None
        """
        assert colname in self.__value_keys
        assert newcolname not in self.__value_keys
        for timeseries in self.__data.values():
            assert newcolname not in timeseries.headers
            timeseries.add_calc_col_single(colname, newcolname, func)
        self.__value_keys.append(newcolname)

    def add_calc_col_full(self, newcolname, func):
        """
        add a new column to every Timeserie, calculated fro existing row data

        parameters:
        newcolname <str> name of new column
        func <func> function to call for every existing row
            lambda <dict> : <float>

        returns:
        None
        """
        assert newcolname not in self.__value_keys
        for timeseries in self.__data.values():
            assert newcolname not in timeseries.headers
            timeseries.add_calc_col_full(newcolname, func)
        self.__value_keys.append(newcolname)

    def remove_col(self, colname):
        """
        remove column named from every Timeserie

        parameters:
        colname <str>

        returns:
        None
        """
        assert colname in self.__value_keys
        for key, timeseries in self.__data.items():
            try:
                assert colname in timeseries.headers
            except AssertionError as exc:
                logging.error("Timeseries with key %s, has no header %s, actually only %s", key, colname, timeseries.headers)
                raise exc
            timeseries.remove_col(colname)
        self.__value_keys.remove(colname)

    def slice(self, colnames):
        ret_data = TimeseriesArray(index_keys=self.__index_keys, value_keys=colnames, ts_key=self.__ts_key)
        for key, value in self.__data.items():
            ret_data.data[key] = value.slice(colnames)
        return ret_data

    def get_group_by_tsa(self, subkeys, group_func):
        """
        subkeys <tuple> tuple of index_keys, must be subset of self.headers
        group_func <func> function to use for aggragation, eg. lambda a: sum(<tuple>)

        return grouped TimeseriesArray from self

        returns <TimeseriesArray>
        """
        # TODO: make sure this object can be aggrgated
        # have alle series, the same row, the same column width and so
        # on
        new_tsa = None
        data = None
        if len(subkeys) > 0:
            for subkey in subkeys:
                try:
                    assert subkey in self.__index_keys
                except AssertionError as exc:
                    logging.error("subkey %s not in headers list of this TimeseriesArray %s", subkey, self.__index_keys)
                    raise exc
            new_tsa = TimeseriesArray(subkeys, self.__value_keys, self.__ts_key)
            data = self.__inflate_keys(subkeys)
        else:
            logging.debug("__total__ aggregation of all timeseries objects into one single Timeseries")
            new_tsa = TimeseriesArray(("__total__", ), self.__value_keys, self.__ts_key)
            data = {
                ("__total__", ) : tuple((timeseries for timeseries in self.__data.values()))
            }
        # make sure every timeserie under each subkey has the same
        # length
        for subkey in data:
            try:
                assert all((len(data[subkey][index]) == len(data[subkey][0]) for index in range(len(data[subkey]))))
            except AssertionError:
                raise StandardError("UnusableDataError Data is not groupable, datalength of timeseries differ")
        self.__aggregate_timeseries(new_tsa, data, group_func)
        return new_tsa

    def get_grouped_tsa(self, subkeys, group_func_name):
        """
        subkeys <tuple> tuple of index_keys, must be subset of self.headers
        group_func_name <str> specifies wich function should be use to aggregate data

        return grouped TimeseriesArray from self

        returns <TimeseriesArray>
        """
        assert group_func_name in self.group_funcs.keys()
        # TODO: make sure this object can be aggrgated
        # have alle series, the same row, the same column width and so
        # on
        new_tsa = None
        data = None
        if len(subkeys) > 0:
            for subkey in subkeys:
                try:
                    assert subkey in self.__index_keys
                except AssertionError as exc:
                    logging.error("subkey %s not in headers list of this TimeseriesArray %s", subkey, self.__index_keys)
                    raise exc
            new_tsa = TimeseriesArray(subkeys, self.__value_keys, self.__ts_key)
            data = self.__inflate_keys(subkeys)
        else:
            logging.debug("__total__ aggregation of all timeseries objects into one single Timeseries")
            new_tsa = TimeseriesArray(("__total__", ), self.__value_keys, self.__ts_key)
            data = {
                ("__total__", ) : tuple((timeseries for timeseries in self.__data.values()))
            }
        # make sure every timeserie under each subkey has the same
        # length
        for subkey in data:
            try:
                assert all((len(data[subkey][index]) == len(data[subkey][0]) for index in range(len(data[subkey]))))
            except AssertionError:
                raise StandardError("UnusableDataError Data is not groupable, datalength of timeseries differ")
        self.__aggregate_timeseries(new_tsa, data, self.group_funcs[group_func_name])
        return new_tsa

    def __inflate_keys(self, subkeys):
        """
        subkeys <tuple> has to be a subset of self.headers
        group_by_func <func> to aggregate a series of values
        drift_range <float> amount of time in s

        group given data by subkey, and use group_func to aggregate data

        so this is a timeseries, there could be a tiny gap of timestamp
        on grouping values, following rules apply to this behaviour

        the first timestamp counts
        timestamps in rnge of +/- drift_range count as the same timestamp
        """
        # aggregate multiple n-Timeseries for unique keys, to a subset
        # of the unique key. the list of timeseries object has to be
        # aggregated afterwards, to get a new TimeseriesArray
        # inflate keys
        ret_data = {}
        for key, timeseries in self.__data.items():
            #logging.debug("working on timeseries for key %s", key)
            index_dict = self.get_index_dict(key)
            #logging.debug("index keys of this timeseries object %s", index_dict)
            new_key_values = []
            for index_key, index_value in index_dict.items():
                if index_key in subkeys:
                    new_key_values.append(index_value)
            new_key_values = tuple(new_key_values)
            #logging.debug("created new subkey %s", new_key_values)
            # add up timeseries to list of this key
            if new_key_values in ret_data:
                ret_data[new_key_values].append(timeseries)
            else:
                ret_data[new_key_values] = [timeseries, ]
        # aggregate timeseries objects with group_by_func
        return ret_data

    @staticmethod
    def __aggregate_timeseries(tsa, data, group_func):
        """
        tsa <TimeseriesArray> usually empty
        data <dict> { tuple of timeseries }  data structure as returned by __inflate_keys
        group_func <func> function which takes tuple and returns single value

        this method aggregates data into tsa and uses group_func to aggragte timeseries objects

        modifies given tsa object
        """
        new_value_keys = list(tsa.value_keys) # TODO create new list, not reference
        new_value_keys.append("group_count")
        for subkey in data.keys():
            #logging.debug("working on subkey %s, there are %s timeseries to aggregate", subkey, len(data[subkey]))
            cols = len(data[subkey][0][0]) # as much columns
            #logging.debug("there are %d columns in every timeseries", cols)
            rows = len(data[subkey][0])
            #logging.debug("there are %d rows in every timeseries", rows)
            timeserie = Timeseries(new_value_keys)
            for rownum in range(rows):
                # define new array, beginning with ts from first timeseries,
                # first row
                row_ts = data[subkey][0][rownum][0]
                rowdata = []
                for colnum in range(1, cols):
                    try:
                        series = tuple((ts[rownum][colnum] for ts in data[subkey]))
                        rowdata.append(group_func(series))
                    except IndexError as exc:
                        logging.exception(exc)
                        logging.error("Timeseries Format Error, series must be skipped")
                if len(rowdata) > 0:
                    rowdata.append(len(data[subkey]))
                    timeserie.add(row_ts, tuple(rowdata))
            assert len(rowdata) == len(new_value_keys)
            tsa.append(subkey, timeserie)

    def sanitize(self, drift=0.05):
        """
        parameters:

        drift <float> defaults to 0.05 and means 5% drift from mean values are allowed

        checks if data is in accurate shape to be grouped
        - data has to start at nearly the same timestamp
        - data has to end at nearly the same timestamp
        - Timeseries has to be the same length
        - the time interval between two consecutive values has to be
          nearly the same

        when i speak from nearly, it means +/- about <drift>%
        """
        avg_len = numpy.median(tuple((len(timeseries) for timeseries in self.__data.values())))
        std_len = numpy.std(tuple((len(timeseries) for timeseries in self.__data.values())))
        logging.info("Average length of stored timeseries: %s", avg_len)
        logging.info("Standard deviation of length       : %s (should be near 0.0)", std_len)
        for key, timeseries in self.__data.items():
            if not len(timeseries) == int(avg_len):
                logging.info(" timeseries with key %s with len %s has not the average data length of %s", key, len(timeseries), avg_len)
                logging.error("deleting key %s: data length difference to high", key)
                del self.__data[key]
        avg_start_ts = numpy.mean(tuple((timeseries[0][0] for timeseries in self.__data.values())))
        logging.info("average start_ts of all timeseries: %s", avg_start_ts)
        for key, timeseries in self.__data.items():
            if not is_near(timeseries[0][0], avg_start_ts, drift):
                logging.info("%s start_ts of %s differs more than 5 percent", key, timeseries[0][0])
                logging.error("deleting key %s: start_ts difference to high", key)
                del self.__data[key]
        avg_stop_ts = numpy.mean(tuple((timeseries[-1][0] for timeseries in self.__data.values())))
        logging.info("average stop_ts of all timeseries: %s", avg_stop_ts)
        for key, timeseries in self.__data.items():
            if not is_near(timeseries[-1][0], avg_stop_ts, drift):
                logging.info("%s stop_ts of %s differs more than 5 percent", key, timeseries[-1][0])
                logging.error("deleting key %s: stop_ts difference to high", key)
                del self.__data[key]
        avg_interval = sum((timeseries.get_interval() for timeseries in self.__data.values())) / len(self.__data)
        std_interval = numpy.std(tuple((timeseries.get_interval() for timeseries in self.__data.values())))
        logging.info("Average interval stored timeseries : %s", avg_interval)
        logging.info("Standard deviation of interval     : %s (should be near 0.0)", std_interval)
        for key, timeseries in self.__data.items():
            if not is_near(timeseries.get_interval(), avg_interval, drift):
                logging.info(" timeseries with key %s with interval %s has not the average data interval of %s", key, timeseries.get_interval(), avg_interval)
                logging.error("deleting key %s: interval difference to high", key)
                del self.__data[key]

    def dump(self):
        """
        function to export all stored data in such a manner, that this is easily used to feed the add function of another TimeseriesArray object

        create new object tsa_new (tsa is the existing one) like

        tsa_new = TimeseriesArray(tsa.index_keys, tsa.value_keys, tsa.ts_key)

        and you can map(tsa.add(), tsa.dump())
        """
        for key, timeseries in self.__data.items():
            # convert key tuple to dict
            key_dict = self.get_index_dict(key)
            # dump timeseries as dictionary and spice dict up with
            # key_dict and timestamp
            ts_dump_dict = timeseries.dump_dict()
            for timestamp in sorted(ts_dump_dict.keys()):
                row = ts_dump_dict[timestamp]
                row.update(key_dict)
                row[self.ts_key] = timestamp
                yield row

    def dump_to_csv(self, filehandle):
        """
        dump all data to filehandle in csv format
        filehandle can also be gzip.open or other types

        parameters:
        filehandle <file>
        """
        outbuffer = []
        outbuffer.append(str((self.__index_keys, self.__value_keys, self.__ts_key)))
        headers = list(self.__index_keys)
        headers.append(self.__ts_key)
        headers.extend(self.__value_keys)
        outbuffer.append(";".join(headers))
        for key, timeseries in self.__data.items():
            # convert key tuple to dict
            for values in timeseries.data:
                row_values = list(key)
                row_values.extend([str(value) for value in values])
                outbuffer.append(";".join(row_values))
        filehandle.write("\n".join(outbuffer))

    def dump_split(self, outpath, overwrite=False):
        """
        dump all data to filehandle in csv format
        filehandle can also be gzip.open or other types

        parameters:
        filehandle <file>
        overwrite <bool> overwrite existing Timeseries files, or not
            the json file is witten nonetheless if this options is set or not
        """
        tsa_filename = self.get_dumpfilename(self.__index_keys)
        logging.debug("tsa_filename: %s", tsa_filename)
        outfile = os.path.join(outpath, tsa_filename)
        outbuffer = {
            "index_keys" : self.__index_keys,
            "value_keys" : self.__value_keys,
            "ts_key" : self.__ts_key,
            "ts_filenames" : []
        }
        for key, timeseries in self.__data.items():
            ts_filename = self.get_ts_dumpfilename(key)
            # skip dump, if file exists, and overwrite=False
            if not os.path.isfile(os.path.join(outpath, ts_filename)) or overwrite:
                logging.debug("dumping key %s to filename %s", key, ts_filename)
                ts_filehandle = gzip.open(os.path.join(outpath, ts_filename), "wb")
                timeseries.dump_to_csv(ts_filehandle)
            outbuffer["ts_filenames"].append(ts_filename)
        json.dump(outbuffer, open(outfile, "wb"))

    @staticmethod
    def load_from_csv(filehandle):
        """
        load data from filehandle, dumped previously with self.dump_to_csv

        returns:
        <TimeseriesArray> object
        """
        data = filehandle.read().split("\n")
        index_keys, value_keys, ts_key = eval(data[0])
        tsa = TimeseriesArray(index_keys, value_keys, ts_key)
        headers = data[1].split(";")
        for row in data[2:]:
            row_dict = dict(zip(headers, row.split(";")))
            tsa.add(row_dict)
        return tsa

    @staticmethod
    def get_ts_dumpfilename(key):
        return "ts_%s.csv.gz" % base64.b64encode(unicode(key))

    @staticmethod
    def get_dumpfilename(index_keys):
        return "tsa_%s.json" % base64.b64encode(unicode(index_keys))

    @staticmethod
    def filtermatch(key_dict, filterkeys, matchtype):
        """
        key_dict is the whole index key, aka
        {hostname : test, instance:1, other:2}

        filterkey is part
        {hostname : test}
        {hostname : test, instance: None, other: None}

        TODO: to be improved
        """
        assert matchtype in ("and", "or")
        matched = 0
        for key in filterkeys.keys():
            if filterkeys[key] is None: # ignore keys with value None
                if matchtype == "and": # and count them as matched
                    matched += 1
                continue
            if key_dict[key] == filterkeys[key]:
                matched += 1
        # every key must match at AND
        if (matchtype == "and") and (matched == len(filterkeys.keys())):
            return True
        # at least one key must match at OR
        elif (matchtype == "or") and (matched > 0):
            return True
        return False

    @staticmethod
    def get_load_split_filenames(path, index_keys, filterkeys=None, matchtype="and"):
        """
        filterkeys could be a part of existing index_keys
        all matching keys will be used
        """
        tsa_filename = TimeseriesArray.get_dumpfilename(index_keys)
        logging.debug("tsa_filename: %s", tsa_filename)
        infile = os.path.join(path, tsa_filename)
        data = json.load(open(infile, "rb"))
        logging.debug("loaded json data")
        logging.debug("index_keys: %s", data["index_keys"])
        logging.debug("value_keys: %s", data["value_keys"])
        logging.debug("ts_key: %s", data["ts_key"])
        logging.debug("number of ts files: %s", len(data["ts_filenames"]))
        filenames = {}
        for filename in data["ts_filenames"]:
            logging.debug("reading ts from file %s", filename)
            enc_key = filename.split(".")[0][3:] # only this pattern ts_(.*).csv.gz
            key = eval(base64.b64decode(enc_key))
            key_dict = dict(zip(index_keys, key))
            if filterkeys is not None:
                if TimeseriesArray.filtermatch(key_dict, filterkeys, matchtype):
                    logging.debug("adding tsa key : %s", key)
                    filenames[key] = os.path.join(path, filename)
            else:
                # no filterkeys means every file is loaded
                logging.debug("adding tsa key : %s", key)
                filenames[key] = os.path.join(path, filename)
        return filenames

    @staticmethod
    def load_split(path, index_keys, filterkeys=None, matchtype="and"):
        """
        filterkeys could be a part of existing index_keys
        all matching keys will be used
        """
        tsa_filename = TimeseriesArray.get_dumpfilename(index_keys)
        infile = os.path.join(path, tsa_filename)
        data = json.load(open(infile, "rb"))
        tsa = TimeseriesArray(data["index_keys"], data["value_keys"], data["ts_key"])
        for key, filename in tsa.get_load_split_filenames(path, index_keys, filterkeys, matchtype).items():
            logging.debug("loading Timeseries from file %s", filename)
            tsa.data[key] = Timeseries.load_from_csv(gzip.open(filename, "rb"))
        return tsa

<commit_msg>Delete TimeseriesArray.py<commit_after><|endoftext|>


<commit_before>import logging
import sys

from gi.repository import Gdk

from lib.args import Args
from lib.config import Config
from lib.clock import Clock
from vocto.port import Port
from vocto.debug import gst_generate_dot

CPU_DECODERS = {
    'h264': 'video/x-h264 ! avdec_h264',
    'jpeg': 'image/jpeg ! jpegdec',
    'mpeg2': 'video/mpeg,mpegversion=2 ! mpeg2dec'
}

VAAPI_DECODERS = {
    'h264': 'vaapih264dec',
    'jpeg': 'vaapijpegdec',
    'mpeg2': 'vaapimpeg2dec',
}

class VideoDisplay(object):
    """Displays a Voctomix-Video-Stream into a GtkWidget"""

    def __init__(self, drawing_area, port, name, width=None, height=None,
                 play_audio=False, level_callback=None):
        self.log = logging.getLogger('VideoDisplay[%u]' % port)
        self.name = name
        self.drawing_area = drawing_area
        self.level_callback = level_callback

        # Setup Server-Connection, Demuxing and Decoding
        pipe = """
                tcpclientsrc
                    name=tcpsrc-{name}
                    host={host}
                    port={port}
                    blocksize=1048576
                ! matroskademux
                    name=demux-{name}
                        """

        if Config.getPreviewsEnabled():
            self.log.info('using encoded previews instead of raw-video')
            if Config.getPreviewVaapi():
                vdec = VAAPI_DECODERS[Config.getPreviewDecoder()]
            else:
                vdec = CPU_DECODERS[Config.getPreviewDecoder()]

            pipe += """
                    demux-{name}.
                    ! queue
                        name=queue-video-{name}
                    ! {vdec}
                    """
        else:
            vdec = None
            self.log.info('using raw-video instead of encoded-previews')
            pipe += """
                    demux-{name}.
                    ! queue
                        name=queue-video-{name}
                    ! {previewcaps}
                    ! {vcaps}
                    """

        if Config.getPreviewNameOverlay() and name:
            textoverlay = """
                        ! textoverlay
                            name=title-{name}
                            text=\"{name}\"
                            valignment=bottom
                            halignment=center
                            shaded-background=yes
                            font-desc="Roboto, 22" """
        else:
            textoverlay = ""

        # Video Display
        videosystem = Config.getVideoSystem()
        self.log.debug('Configuring for Video-System %s', videosystem)
        if videosystem == 'gl':
            pipe += textoverlay + """
                                ! glupload
                                ! glcolorconvert
                                ! glimagesinkelement
                                    name=imagesink-{name}
                                """.format(name=name)

        elif videosystem == 'xv':
            pipe += textoverlay + """
                                ! xvimagesink
                                    name=imagesink-{name}
                                """.format(name=name)

        elif videosystem == 'x':
            prescale_caps = 'video/x-raw'
            if width and height:
                prescale_caps += ',width=%u,height=%u' % (width, height)

            pipe += """
                    ! videoconvert
                    ! videoscale {textoverlay}
                    ! {prescale_caps}
                    ! ximagesink
                        name=imagesink-{name}
                    """.format(
                                    prescale_caps=prescale_caps,
                                    textoverlay=textoverlay,
                                    name=name)
        else:
            raise Exception(
                'Invalid Videodisplay-System configured: %s' % videosystem
            )

        # If Playback is requested, push fo pulseaudio
        if play_audio:
            # add an Audio-Path through a level-Element
            pipe += """
                    demux-{name}.
                    ! queue
                        name=queue-audio-{name}
                    ! {acaps}
                    ! level
                        name=lvl
                        interval=50000000
                    ! audioconvert
                    ! pulsesink
                        name=audiosink-{name}"""

        pipe = pipe.format(
            name=name,
            acaps=Config.getAudioCaps(),
            vcaps=Config.getVideoCaps(),
            previewcaps=Config.getPreviewCaps(),
            host=Config.getHost(),
            vdec=vdec,
            port=port,
        )

        self.log.info('Creating Display-Pipeline:\n%s', pipe)
        try:
            # launch gstreamer pipeline
            self.pipeline = Gst.parse_launch(pipe)
            self.log.info("pipeline launched successfuly")
        except:
            self.log.error("Can not launch pipeline")
            sys.exit(-1)

        if Args.dot:
            self.log.debug('Generating DOT image of videodisplay pipeline')
            gst_generate_dot(self.pipeline, "gui.videodisplay.{}".format(name))

        self.pipeline.use_clock(Clock)

        self.drawing_area.add_events(
            Gdk.EventMask.KEY_PRESS_MASK | Gdk.EventMask.KEY_RELEASE_MASK)
        self.drawing_area.connect("realize", self.on_realize)
        self.drawing_area.realize()

        bus = self.pipeline.get_bus()
        bus.add_signal_watch()
        bus.enable_sync_message_emission()

        bus.connect('message::error', self.on_error)
        bus.connect("sync-message::element", self.on_syncmsg)
        self.pipeline.bus.connect(
            "message::state-changed", self.on_state_changed)

        if self.level_callback:
            bus.connect("message::element", self.on_level)

    def on_realize(self, win):
        self.imagesink = self.pipeline.get_by_name(
            'imagesink-{name}'.format(name=self.name))
        self.xid = self.drawing_area.get_property('window').get_xid()
        self.log.debug('Realized Drawing-Area with xid %u', self.xid)

        self.log.debug('Launching Display-Pipeline')
        self.pipeline.set_state(Gst.State.PLAYING)

    def on_syncmsg(self, bus, msg):
        if type(msg) == Gst.Message and self.imagesink:
            if msg.get_structure().get_name() == "prepare-window-handle":
                self.log.info(
                    'Setting imagesink window-handle to %s', self.xid)
                self.imagesink.set_window_handle(self.xid)

    def on_error(self, bus, message):
        (error, debug) = message.parse_error()
        self.log.error(
            "GStreamer pipeline element '%s' signaled an error #%u: %s" % (message.src.name, error.code, error.message))

    def mute(self, mute):
        self.pipeline.get_by_name("audiosink-{name}".format(name=self.name)).set_property(
            "volume", 1 if mute else 0)

    def on_level(self, bus, msg):
        if msg.src.name != 'lvl':
            return

        if msg.type != Gst.MessageType.ELEMENT:
            return

        rms = msg.get_structure().get_value('rms')
        peak = msg.get_structure().get_value('peak')
        decay = msg.get_structure().get_value('decay')
        self.level_callback(rms, peak, decay)

    def on_state_changed(self, bus, message):
        if message.parse_state_changed().newstate == Gst.State.PLAYING:
            self.drawing_area.show()
<commit_msg>univified the behavior of the different displaymodes cleanup<commit_after>import logging
import sys

from gi.repository import Gst, Gdk

from lib.args import Args
from lib.config import Config
from lib.clock import Clock
from vocto.port import Port
from vocto.debug import gst_generate_dot

CPU_DECODERS = {
    'h264': 'video/x-h264 ! avdec_h264',
    'jpeg': 'image/jpeg ! jpegdec',
    'mpeg2': 'video/mpeg,mpegversion=2 ! mpeg2dec'
}

VAAPI_DECODERS = {
    'h264': 'vaapih264dec',
    'jpeg': 'vaapijpegdec',
    'mpeg2': 'vaapimpeg2dec',
}

class VideoDisplay(object):
    """Displays a Voctomix-Video-Stream into a GtkWidget"""

    def __init__(self, drawing_area, port, name, width=None, height=None,
                 play_audio=False, level_callback=None):
        self.log = logging.getLogger('VideoDisplay[%u]' % port)
        self.name = name
        self.drawing_area = drawing_area
        self.level_callback = level_callback
        video_decoder = None

        # Setup Server-Connection, Demuxing and Decoding
        pipe = """
                tcpclientsrc
                    name=tcpsrc-{name}
                    host={host}
                    port={port}
                    blocksize=1048576
                ! matroskademux
                    name=demux-{name}
                """.format(name=name,
                           host=Config.getHost(),
                           port=port)

        if Config.getPreviewsEnabled():
            self.log.info('using encoded previews instead of raw-video')
            if Config.getPreviewVaapi():
                video_decoder = VAAPI_DECODERS[Config.getPreviewDecoder()]
            else:
                video_decoder = CPU_DECODERS[Config.getPreviewDecoder()]

            pipe += """
                    demux-{name}.
                    ! queue
                        name=queue-video-{name}
                    ! {video_decoder}
                    """.format(name=name,
                               video_decoder=video_decoder)
        else:
            video_decoder = None
            preview_caps = 'video/x-raw'
            self.log.info('using raw-video instead of encoded-previews')
            pipe += """
                    demux-{name}.
                    ! queue
                        name=queue-video-{name}
                    ! {previewcaps}
                    ! {vcaps}
                    """.format(name=name,
                               previewcaps=preview_caps,
                               vcaps=Config.getVideoCaps())

        if Config.getPreviewNameOverlay() and name:
            textoverlay = """
                        ! textoverlay
                            name=title-{name}
                            text=\"{name}\"
                            valignment=bottom
                            halignment=center
                            shaded-background=yes
                            font-desc="Roboto, 22"
                        """.format(name=name)
        else:
            textoverlay = ""
        pipe += textoverlay

        # Video Display
        videosystem = Config.getVideoSystem()
        self.log.debug('Configuring for Video-System %s', videosystem)

        pipe += """
                ! videoconvert
                ! videoscale
                """

        if videosystem == 'gl':
            pipe += """
                    ! glupload
                    ! glcolorconvert
                    ! glimagesinkelement
                        name=imagesink-{name}
                    """.format(name=name)

        elif videosystem == 'xv':
            pipe += """
                    ! xvimagesink
                        name=imagesink-{name}
                    """.format(name=name)

        elif videosystem == 'x':
            #if width and height:
                #prescale_caps = 'width=%u,height=%u' % (width, height)
                #pipe += """
                #    ! {prescale_caps}
                #    """.format(prescale_caps=prescale_caps)

            pipe += """
                ! ximagesink
                    name=imagesink-{name}
                """.format(name=name)
        else:
            raise Exception(
                'Invalid Videodisplay-System configured: %s' % videosystem
            )

        # If Playback is requested, push fo pulseaudio
        if play_audio:
            # add an Audio-Path through a level-Element
            pipe += """
                    demux-{name}.
                    ! queue
                        name=queue-audio-{name}
                    ! {acaps}
                    ! level
                        name=lvl
                        interval=50000000
                    ! audioconvert
                    ! pulsesink
                        name=audiosink-{name}"""

        pipe = pipe.format(name=name,
                           acaps=Config.getAudioCaps(),
                           port=port,
                           )

        self.log.info('Creating Display-Pipeline:\n%s', pipe)
        try:
            # launch gstreamer pipeline
            self.pipeline = Gst.parse_launch(pipe)
            self.log.info("pipeline launched successfuly")
        except:
            self.log.error("Can not launch pipeline")
            sys.exit(-1)

        if Args.dot:
            self.log.debug('Generating DOT image of videodisplay pipeline')
            gst_generate_dot(self.pipeline, "gui.videodisplay.{}".format(name))

        self.pipeline.use_clock(Clock)

        self.drawing_area.add_events(
            Gdk.EventMask.KEY_PRESS_MASK | Gdk.EventMask.KEY_RELEASE_MASK)
        self.drawing_area.connect("realize", self.on_realize)
        self.drawing_area.realize()

        bus = self.pipeline.get_bus()
        bus.add_signal_watch()
        bus.enable_sync_message_emission()

        bus.connect('message::error', self.on_error)
        bus.connect("sync-message::element", self.on_syncmsg)
        self.pipeline.bus.connect(
            "message::state-changed", self.on_state_changed)

        if self.level_callback:
            bus.connect("message::element", self.on_level)

    def on_realize(self, win):
        self.imagesink = self.pipeline.get_by_name(
            'imagesink-{name}'.format(name=self.name))
        self.xid = self.drawing_area.get_property('window').get_xid()
        self.log.debug('Realized Drawing-Area with xid %u', self.xid)

        self.log.debug('Launching Display-Pipeline')
        self.pipeline.set_state(Gst.State.PLAYING)

    def on_syncmsg(self, bus, msg):
        if type(msg) == Gst.Message and self.imagesink:
            if msg.get_structure().get_name() == "prepare-window-handle":
                self.log.info(
                    'Setting imagesink window-handle to %s', self.xid)
                self.imagesink.set_window_handle(self.xid)

    def on_error(self, bus, message):
        (error, debug) = message.parse_error()
        self.log.error(
            "GStreamer pipeline element '%s' signaled an error #%u: %s" % (message.src.name, error.code, error.message))

    def mute(self, mute):
        self.pipeline.get_by_name("audiosink-{name}".format(name=self.name)).set_property(
            "volume", 1 if mute else 0)

    def on_level(self, bus, msg):
        if msg.src.name != 'lvl':
            return

        if msg.type != Gst.MessageType.ELEMENT:
            return

        rms = msg.get_structure().get_value('rms')
        peak = msg.get_structure().get_value('peak')
        decay = msg.get_structure().get_value('decay')
        self.level_callback(rms, peak, decay)

    def on_state_changed(self, bus, message):
        if message.parse_state_changed().newstate == Gst.State.PLAYING:
            self.drawing_area.show()
<|endoftext|>


<commit_before>#! /usr/bin/env python
#*******************************************************************************
# ALMA - Atacama Large Millimiter Array
# (c) National Research Council of Canada, 2008
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2.1 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307  USA
#
# "@(#) $Id: ASI.py,v 1.1 2008/10/09 16:11:09 agrimstrup Exp $"
#
# who       when      what
# --------  --------  ----------------------------------------------
# agrimstrup  2008-10-02  created
#

ASI_VERSION = "0.9"
ALARMS_TOPIC = "CMW.ALARM_SYSTEM.ALARMS.SOURCES"
ALARM_SOURCE_NAME = "ALARM_SYSTEM_SOURCES"
BACKUP_DELIVERY_MODE = 0
BACKUP_PRIORITY = 9
BACKUP_TIME_TO_LIVE = 60000
CHANGES_DELIVERY_MODE = 0
CHANGES_PRIORITY = 9
CHANGES_TIME_TO_LIVE = 60000

class ASIConfiguration(object):
    """
    Python class that encapsulates various configuration data for the LASER
    alarm system for use by Python alarm source clients.
    """
    def __init__(self):
        self.asiVersion = ASI_VERSION
        self.alarmsTopic = ALARMS_TOPIC
        self.backupDeliveryMode = BACKUP_DELIVERY_MODE
        self.backupPriority = BACKUP_PRIORITY
        self.backupTimeToLive = BACKUP_TIME_TO_LIVE
        self.changesDeliveryMode = CHANGES_DELIVERY_MODE
        self.changesPriority = CHANGES_PRIORITY
        self.changesTimeToLive = CHANGES_TIME_TO_LIVE

class ASIMessage(object):
    """
    Class that encapsulates one or more fault states to be sent to the LASER
    alarm server.
    """
    def __init__(self, states = None):
        self.faultStates = states
        self.backup = False
        self.version = ASI_VERSION
        self.sourceName = ALARM_SOURCE_NAME
        self.sourceHostname = None
        self.sourceTimestamp = None

    def toXML(self):
        """
        Build the XML representation of the message which will be sent to the alarm server.
        """
        pad = '\t'.expandtabs(3)
        taglist = []
        header = '<?xml version="1.0" encoding="ISO-8859-1"?>\n'
        taglist.append('<ASI-message xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" backup="%s" version="%s" xsi:type="ASI-message">\n' % (str(self.backup).lower(), self.version))
        taglist.append('<source-name>%s</source_name>\n' % self.sourceName)
        taglist.append('<source-hostname>%s</source_hostname>\n' % self.sourceHostname)
        taglist.append(self.sourceTimestamp.toXML('source-timestamp', 0))
        taglist.append('<fault-states>\n')
        try:
            for fs in self.faultStates:
                taglist.append(fs.toXML(amountToIndent=6))
        except TypeError:
            pass
        taglist.append('</fault-states>\n')
        return pad.join(taglist) + '</ASI-message>\n'

#
# ___oOo___
<commit_msg>The XML header was missing from the string generated by ASIMessage. There were a couple of typos in the tags as well.<commit_after>#! /usr/bin/env python
#*******************************************************************************
# ALMA - Atacama Large Millimiter Array
# (c) National Research Council of Canada, 2008
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2.1 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307  USA
#
# "@(#) $Id: ASI.py,v 1.2 2008/10/29 16:20:18 agrimstrup Exp $"
#
# who       when      what
# --------  --------  ----------------------------------------------
# agrimstrup  2008-10-02  created
#

ASI_VERSION = "0.9"
ALARMS_TOPIC = "CMW.ALARM_SYSTEM.ALARMS.SOURCES"
ALARM_SOURCE_NAME = "ALARM_SYSTEM_SOURCES"
BACKUP_DELIVERY_MODE = 0
BACKUP_PRIORITY = 9
BACKUP_TIME_TO_LIVE = 60000
CHANGES_DELIVERY_MODE = 0
CHANGES_PRIORITY = 9
CHANGES_TIME_TO_LIVE = 60000

class ASIConfiguration(object):
    """
    Python class that encapsulates various configuration data for the LASER
    alarm system for use by Python alarm source clients.
    """
    def __init__(self):
        self.asiVersion = ASI_VERSION
        self.alarmsTopic = ALARMS_TOPIC
        self.backupDeliveryMode = BACKUP_DELIVERY_MODE
        self.backupPriority = BACKUP_PRIORITY
        self.backupTimeToLive = BACKUP_TIME_TO_LIVE
        self.changesDeliveryMode = CHANGES_DELIVERY_MODE
        self.changesPriority = CHANGES_PRIORITY
        self.changesTimeToLive = CHANGES_TIME_TO_LIVE

class ASIMessage(object):
    """
    Class that encapsulates one or more fault states to be sent to the LASER
    alarm server.
    """
    def __init__(self, states = None):
        self.faultStates = states
        self.backup = False
        self.version = ASI_VERSION
        self.sourceName = ALARM_SOURCE_NAME
        self.sourceHostname = None
        self.sourceTimestamp = None

    def toXML(self):
        """
        Build the XML representation of the message which will be sent to the alarm server.
        """
        pad = '\t'.expandtabs(3)
        taglist = []
        header = '<?xml version="1.0" encoding="ISO-8859-1"?>\n'
        taglist.append('<ASI-message xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" backup="%s" version="%s" xsi:type="ASI-message">\n' % (str(self.backup).lower(), self.version))
        taglist.append('<source-name>%s</source-name>\n' % self.sourceName)
        taglist.append('<source-hostname>%s</source-hostname>\n' % self.sourceHostname)
        taglist.append(self.sourceTimestamp.toXML('source-timestamp', 0))
        taglist.append('<fault-states>\n')
        try:
            for fs in self.faultStates:
                taglist.append(fs.toXML(amountToIndent=6))
        except TypeError:
            pass
        taglist.append('</fault-states>\n')
        return header + pad.join(taglist) + '</ASI-message>\n'

#
# ___oOo___
<|endoftext|>


<commit_before>"""
:copyright: (c) 2012 - 2015 by Satoru SATOH <ssato@redhat.com>
:license: BSD-3
"""
from __future__ import absolute_import
from __future__ import print_function

import codecs
import glob
import logging
import os.path
import os
import sys

import anytemplate.compat

try:
    from anyconfig.api import container, load
except ImportError:
    container = dict

    try:
        import json
    except ImportError:
        try:
            import simplejson as json
        except ImportError:
            raise ("Could not import any json module to load contexts!"
                   " Aborting...")

    def load(filepath, _ftype):
        return json.load(open(filepath))

try:
    from anyconfig.utils import get_file_extension
    get_file_extension
except ImportError:
    def _get_file_extension(filepath):
        """
        >>> get_file_extension("/a/b/c")
        ''
        >>> get_file_extension("/a/b.txt")
        'txt'
        >>> get_file_extension("/a/b/c.tar.xz")
        'xz'
        """
        _ext = os.path.splitext(filepath)[-1]
        if _ext:
            return _ext[1:] if _ext.startswith('.') else _ext
        else:
            return ''

LOGGER = logging.getLogger(__name__)

def get_locale_sensitive_stdout(encoding=anytemplate.compat.ENCODING):
    """
    :param encoding: Chart sets encoding
    :return: sys.stdout can output encoded strings

    >>> _t = get_locale_sensitive_stdout("UTF-8")
    """
    return codecs.getwriter(encoding)(sys.stdout)

def get_locale_sensitive_stdin(encoding=anytemplate.compat.ENCODING):
    """
    :param encoding: Chart sets encoding
    :return: sys.stdout can output encoded strings

    >>> _t = get_locale_sensitive_stdin("UTF-8")
    """
    return codecs.getreader(encoding)(sys.stdin)

def uniq(xs):
    """Remove duplicates in given list with its order kept.

    >>> uniq([])
    []
    >>> uniq([1, 4, 5, 1, 2, 3, 5, 10])
    [1, 4, 5, 2, 3, 10]
    """
    acc = xs[:1]
    for x in xs[1:]:
        if x not in acc:
            acc += [x]

    return acc

def chaincalls(callables, x):
    """
    :param callables: callable objects to apply to x in this order
    :param x: Object to apply callables

    >>> chaincalls([lambda a: a + 1, lambda b: b + 2], 0)
    3
    """
    for c in callables:
        assert callable(c), "%s is not callable object!" % str(c)
        x = c(x)

    return x

def normpath(path):
    """Normalize given path in various different forms.

    >>> normpath("/tmp/../etc/hosts")
    '/etc/hosts'
    >>> normpath("~root/t")
    '/root/t'
    """
    if "~" in path:
        fs = [os.path.expanduser, os.path.normpath, os.path.abspath]
    else:
        fs = [os.path.normpath, os.path.abspath]

    return chaincalls(fs, path)

def flip(xy):
    """
    :param xy: A tuple of pair items

    >>> flip((1, 2))
    (2, 1)
    """
    (x, y) = xy
    return (y, x)

def concat(xss):
    """
    >>> concat([[]])
    []
    >>> concat((()))
    []
    >>> concat([[1,2,3],[4,5]])
    [1, 2, 3, 4, 5]
    >>> concat([[1,2,3],[4,5,[6,7]]])
    [1, 2, 3, 4, 5, [6, 7]]
    >>> concat(((1,2,3),(4,5,[6,7])))
    [1, 2, 3, 4, 5, [6, 7]]
    >>> concat(((1,2,3),(4,5,[6,7])))
    [1, 2, 3, 4, 5, [6, 7]]
    >>> concat((i, i*2) for i in range(3))
    [0, 0, 1, 2, 2, 4]
    """
    return list(anytemplate.compat.from_iterable(xs for xs in xss))

def parse_filespec(fspec, sep=':', gpat='*'):
    """
    Parse given filespec `fspec` and return [(filetype, filepath)].

    Because anyconfig.api.load should find correct file's type to load by the
    file extension, this function will not try guessing file's type if not file
    type is specified explicitly.

    :param fspec: filespec
    :param sep: a char separating filetype and filepath in filespec
    :param gpat: a char for glob pattern

    >>> parse_filespec("base.json")
    [('base.json', None)]
    >>> parse_filespec("json:base.json")
    [('base.json', 'json')]
    >>> parse_filespec("yaml:foo.yaml")
    [('foo.yaml', 'yaml')]
    >>> parse_filespec("yaml:foo.dat")
    [('foo.dat', 'yaml')]

    # FIXME: How to test this?
    # >>> parse_filespec("yaml:bar/*.conf")
    # [('bar/a.conf', 'yaml'), ('bar/b.conf', 'yaml')]

    TODO: Allow '*' (glob pattern) in filepath when escaped with '\\', etc.
    """
    tp = (ft, fp) = tuple(fspec.split(sep)) if sep in fspec else (None, fspec)

    return [(fs, ft) for fs in sorted(glob.glob(fp))] \
        if gpat in fspec else [flip(tp)]

def parse_and_load_contexts(contexts, werr=False,
                            enc=anytemplate.compat.ENCODING):
    """
    :param contexts: list of context file specs
    :param werr: Exit immediately if True and any errors occurrs
        while loading context files
    :param enc: Input encoding of context files (dummy param)
    """
    ctx = container()

    if contexts:
        for fpath, ftype in concat(parse_filespec(f) for f in contexts):
            diff = load(fpath, ftype)
            ctx.update(diff)

    return ctx

def write_to_output(content, output=None,
                    encoding=anytemplate.compat.ENCODING):
    if output and not output == '-':
        outdir = os.path.dirname(output)
        if outdir and not os.path.exists(outdir):
            os.makedirs(outdir)

        anytemplate.compat.copen(output, 'w').write(content)
    else:
        print(content, get_locale_sensitive_stdout())

def mk_template_paths(filepath, template_paths=None):
    """
    Make template paths from given filepath and paths list.

    :param filepath: (Base) filepath of template file
    :param template_paths: Template search paths

    >>> fn = __file__
    >>> fdir = os.path.abspath(os.path.dirname(fn))
    >>> assert mk_template_paths(fn, []) == ['.', fdir]
    >>> assert mk_template_paths(fn, ["/etc", ]) == ["/etc", fdir]
    """
    tmpldir = os.path.abspath(os.path.dirname(filepath))
    if template_paths is None or not template_paths:
        return [os.curdir, tmpldir]  # default:
    else:
        return uniq(template_paths + [tmpldir])

def find_template_from_path(filepath, paths=None):
    """
    Return resolved path of given template file

    :param filepath: (Base) filepath of template file
    :param paths: Template search paths
    """
    if paths is None or not paths:
        paths = [os.path.dirname(filepath), os.curdir]

    for p in paths:
        candidate = os.path.join(p, filepath)
        if os.path.exists(candidate):
            return candidate

    LOGGER.warn("Could not find template=%s in paths=%s", filepath, paths)
    return None

# vim:sw=4:ts=4:et:
<commit_msg>make anytemplate.utils.mk_template_paths can process cases if filepath is None<commit_after>"""
:copyright: (c) 2012 - 2015 by Satoru SATOH <ssato@redhat.com>
:license: BSD-3
"""
from __future__ import absolute_import
from __future__ import print_function

import codecs
import glob
import logging
import os.path
import os
import sys

import anytemplate.compat

try:
    from anyconfig.api import container, load
except ImportError:
    container = dict

    try:
        import json
    except ImportError:
        try:
            import simplejson as json
        except ImportError:
            raise ("Could not import any json module to load contexts!"
                   " Aborting...")

    def load(filepath, _ftype):
        return json.load(open(filepath))

try:
    from anyconfig.utils import get_file_extension
    get_file_extension
except ImportError:
    def _get_file_extension(filepath):
        """
        >>> get_file_extension("/a/b/c")
        ''
        >>> get_file_extension("/a/b.txt")
        'txt'
        >>> get_file_extension("/a/b/c.tar.xz")
        'xz'
        """
        _ext = os.path.splitext(filepath)[-1]
        if _ext:
            return _ext[1:] if _ext.startswith('.') else _ext
        else:
            return ''

LOGGER = logging.getLogger(__name__)

def get_locale_sensitive_stdout(encoding=anytemplate.compat.ENCODING):
    """
    :param encoding: Chart sets encoding
    :return: sys.stdout can output encoded strings

    >>> _t = get_locale_sensitive_stdout("UTF-8")
    """
    return codecs.getwriter(encoding)(sys.stdout)

def get_locale_sensitive_stdin(encoding=anytemplate.compat.ENCODING):
    """
    :param encoding: Chart sets encoding
    :return: sys.stdout can output encoded strings

    >>> _t = get_locale_sensitive_stdin("UTF-8")
    """
    return codecs.getreader(encoding)(sys.stdin)

def uniq(xs):
    """Remove duplicates in given list with its order kept.

    >>> uniq([])
    []
    >>> uniq([1, 4, 5, 1, 2, 3, 5, 10])
    [1, 4, 5, 2, 3, 10]
    """
    acc = xs[:1]
    for x in xs[1:]:
        if x not in acc:
            acc += [x]

    return acc

def chaincalls(callables, x):
    """
    :param callables: callable objects to apply to x in this order
    :param x: Object to apply callables

    >>> chaincalls([lambda a: a + 1, lambda b: b + 2], 0)
    3
    """
    for c in callables:
        assert callable(c), "%s is not callable object!" % str(c)
        x = c(x)

    return x

def normpath(path):
    """Normalize given path in various different forms.

    >>> normpath("/tmp/../etc/hosts")
    '/etc/hosts'
    >>> normpath("~root/t")
    '/root/t'
    """
    if "~" in path:
        fs = [os.path.expanduser, os.path.normpath, os.path.abspath]
    else:
        fs = [os.path.normpath, os.path.abspath]

    return chaincalls(fs, path)

def flip(xy):
    """
    :param xy: A tuple of pair items

    >>> flip((1, 2))
    (2, 1)
    """
    (x, y) = xy
    return (y, x)

def concat(xss):
    """
    >>> concat([[]])
    []
    >>> concat((()))
    []
    >>> concat([[1,2,3],[4,5]])
    [1, 2, 3, 4, 5]
    >>> concat([[1,2,3],[4,5,[6,7]]])
    [1, 2, 3, 4, 5, [6, 7]]
    >>> concat(((1,2,3),(4,5,[6,7])))
    [1, 2, 3, 4, 5, [6, 7]]
    >>> concat(((1,2,3),(4,5,[6,7])))
    [1, 2, 3, 4, 5, [6, 7]]
    >>> concat((i, i*2) for i in range(3))
    [0, 0, 1, 2, 2, 4]
    """
    return list(anytemplate.compat.from_iterable(xs for xs in xss))

def parse_filespec(fspec, sep=':', gpat='*'):
    """
    Parse given filespec `fspec` and return [(filetype, filepath)].

    Because anyconfig.api.load should find correct file's type to load by the
    file extension, this function will not try guessing file's type if not file
    type is specified explicitly.

    :param fspec: filespec
    :param sep: a char separating filetype and filepath in filespec
    :param gpat: a char for glob pattern

    >>> parse_filespec("base.json")
    [('base.json', None)]
    >>> parse_filespec("json:base.json")
    [('base.json', 'json')]
    >>> parse_filespec("yaml:foo.yaml")
    [('foo.yaml', 'yaml')]
    >>> parse_filespec("yaml:foo.dat")
    [('foo.dat', 'yaml')]

    # FIXME: How to test this?
    # >>> parse_filespec("yaml:bar/*.conf")
    # [('bar/a.conf', 'yaml'), ('bar/b.conf', 'yaml')]

    TODO: Allow '*' (glob pattern) in filepath when escaped with '\\', etc.
    """
    tp = (ft, fp) = tuple(fspec.split(sep)) if sep in fspec else (None, fspec)

    return [(fs, ft) for fs in sorted(glob.glob(fp))] \
        if gpat in fspec else [flip(tp)]

def parse_and_load_contexts(contexts, werr=False,
                            enc=anytemplate.compat.ENCODING):
    """
    :param contexts: list of context file specs
    :param werr: Exit immediately if True and any errors occurrs
        while loading context files
    :param enc: Input encoding of context files (dummy param)
    """
    ctx = container()

    if contexts:
        for fpath, ftype in concat(parse_filespec(f) for f in contexts):
            diff = load(fpath, ftype)
            ctx.update(diff)

    return ctx

def write_to_output(content, output=None,
                    encoding=anytemplate.compat.ENCODING):
    if output and not output == '-':
        outdir = os.path.dirname(output)
        if outdir and not os.path.exists(outdir):
            os.makedirs(outdir)

        anytemplate.compat.copen(output, 'w').write(content)
    else:
        print(content, get_locale_sensitive_stdout())

def mk_template_paths(filepath=None, template_paths=None):
    """
    Make template paths from given filepath and paths list.

    :param filepath: (Base) filepath of template file
    :param template_paths: Template search paths

    >>> fn = __file__
    >>> fdir = os.path.abspath(os.path.dirname(fn))
    >>> assert mk_template_paths(fn, []) == ['.', fdir]
    >>> assert mk_template_paths(fn, ["/etc", ]) == ["/etc", fdir]
    >>> assert mk_template_paths(None, ["/etc", ]) == [os.curdir]
    """
    if filepath is None or not filepath:
        return [os.curdir]

    tmpldir = os.path.abspath(os.path.dirname(filepath))
    if template_paths is None or not template_paths:
        return [os.curdir, tmpldir]  # default:
    else:
        return uniq(template_paths + [tmpldir])

def find_template_from_path(filepath, paths=None):
    """
    Return resolved path of given template file

    :param filepath: (Base) filepath of template file
    :param paths: Template search paths
    """
    if paths is None or not paths:
        paths = [os.path.dirname(filepath), os.curdir]

    for p in paths:
        candidate = os.path.join(p, filepath)
        if os.path.exists(candidate):
            return candidate

    LOGGER.warn("Could not find template=%s in paths=%s", filepath, paths)
    return None

# vim:sw=4:ts=4:et:
<|endoftext|>


<commit_before>import geocoder

location = 'Canada'
ip = '74.125.226.99'

def test_entry_points():
    geocoder.ip
    geocoder.osm
    geocoder.bing
    geocoder.nokia
    geocoder.google
    geocoder.tomtom
    geocoder.reverse
    geocoder.mapquest

def test_google():
    g = geocoder.google(location)
    assert g.ok

def test_bing():
    ok = False
    for i in xrange(3):
        g = geocoder.bing(location)
        if g.ok:
            ok = True
    assert ok

def test_osm():
    g = geocoder.osm(location)
    assert g.ok

def test_tomtom():
    g = geocoder.tomtom(location)
    assert g.ok

def test_arcgis():
    g = geocoder.arcgis(location)
    assert g.ok

def test_mapquest():
    g = geocoder.mapquest(location)
    assert g.ok

def test_reverse():
    g = geocoder.ip(ip)
    assert g.ok
<commit_msg>Removed HTTP requests from Testing<commit_after>import geocoder

location = 'Canada'
ip = '74.125.226.99'

def test_entry_points():
    geocoder.ip
    geocoder.osm
    geocoder.bing
    geocoder.nokia
    geocoder.google
    geocoder.tomtom
    geocoder.reverse
    geocoder.mapquest

def test_google():
    pass

def test_bing():
    pass

def test_osm():
    pass

def test_tomtom():
    pass

def test_arcgis():
    pass

def test_mapquest():
    pass

def test_reverse():
    pass
<|endoftext|>


<commit_before>from django.test import TestCase
from django.contrib.auth.models import User
from django.conf import settings
from pandas import read_csv, notnull, to_datetime

from rhizome.etl_tasks.simple_upload_transform import SimpleDocTransform
from rhizome.models import *

class TransformUploadTestCase(TestCase):

    def __init__(self, *args, **kwargs):

        super(TransformUploadTestCase, self).__init__(*args, **kwargs)

    def set_up(self):

        self.create_metadata()
        self.user = User.objects.get(username = 'test')
        self.document = Document.objects.get(doc_title = 'test')

        self.test_file_location = 'eoc_post_campaign.csv'
        self.document.docfile = self.test_file_location
        self.document.save()

        self.location_list = Location.objects.all().values_list('name',flat=True)

    def test_simple_transform(self):

        self.set_up()

        sdt = SimpleDocTransform(self.user.id, self.document.id)
        sdt.main()

        the_value_from_the_database = DataPointComputed.objects.get(
                campaign_id = self.mapped_campaign_id,
                indicator_id = self.mapped_indicator_with_data,
                location_id = self.mapped_location_id
            ).value

        some_cell_value_from_the_file = 0.082670906
        ## find this from the data frame by selecting the cell where we have mapped the data..

        self.assertEqual(some_cell_value_from_the_file, the_value_from_the_database)

    def create_metadata(self):
        '''
        Creating the Indicator, location, Campaign, meta data needed for the
        system to aggregate / caclulate.
        '''

        top_lvl_tag = IndicatorTag.objects.create(id = 1, tag_name='Polio')

        campaign_df = read_csv('rhizome/tests/_data/campaigns.csv')
        campaign_df['top_lvl_indicator_tag_id'] = top_lvl_tag.id

        campaign_df['start_date'] = to_datetime(campaign_df['start_date'])
        campaign_df['end_date'] = to_datetime(campaign_df['end_date'])

        location_df= read_csv('rhizome/tests/_data/locations.csv')
        indicator_df = read_csv('rhizome/tests/_data/indicators.csv')

        user_id = User.objects.create_user('test','test@test.com', 'test').id
        office_id = Office.objects.create(id=1,name='test').id

        cache_job_id = CacheJob.objects.create(id = -2, \
            date_attempted = '2015-01-01',is_error = False)

        document_id = Document.objects.create(
            doc_title = 'test',
            file_header = 'Campaign,Wardcode,uq_id,HHsampled,HHvisitedTEAMS,Marked0to59,UnImmun0to59,NOimmReas1,NOimmReas2,NOimmReas3,NOimmReas4,NOimmReas5,NOimmReas6,NOimmReas7,NOimmReas8,NOimmReas9,NOimmReas10,NOimmReas11,NOimmReas12,NOimmReas13,NOimmReas14,NOimmReas15,NOimmReas16,NOimmReas17,NOimmReas18,NOimmReas19,NOimmReas20,ZeroDose,TotalYoungest,YoungstRI,RAssessMrk,RCorctCAT,RIncorect,RXAssessMrk,RXCorctCAT,RXIncorect,STannounc,SRadio,STradlead,SReiliglead,SMosque,SNewspaper,SPoster,Sbanner,SRelative,SHworker,Scommmob,SNOTAWARE,Influence1,Influence2,Influence3,Influence4,Influence5,Influence6,Influence7,Influence8',
            created_by_id = user_id,
            guid = 'test').id

        campaign_type = CampaignType.objects.create(id=1,name="test")

        locations = self.model_df_to_data(location_df,Location)
        campaigns = self.model_df_to_data(campaign_df,Campaign)
        indicators = self.model_df_to_data(indicator_df,Indicator)

        self.mapped_location_id = locations[0].id
        loc_map = SourceObjectMap.objects.create(
            source_object_code = 'AF001039003000000000',
            content_type = 'location',
            mapped_by_id = user_id,
            master_object_id = self.mapped_location_id
        )

        source_campaign_string = '2016 March NID OPV'
        self.mapped_campaign_id = campaigns[0].id
        campaign_map = SourceObjectMap.objects.create(
            source_object_code = source_campaign_string,
            content_type = 'campaign',
            mapped_by_id = user_id,
            master_object_id = self.mapped_campaign_id
        )
        self.mapped_indicator_id_0 = locations[0].id
        indicator_map = SourceObjectMap.objects.create(
            source_object_code = 'Percent missed children_PCA',
            content_type = 'indicator',
            mapped_by_id = user_id,
            master_object_id = self.mapped_indicator_id_0
        )

        self.mapped_indicator_id_1 = locations[1].id
        indicator_map = SourceObjectMap.objects.create(
            source_object_code = 'Percent missed due to not visited',
            content_type = 'indicator',
            mapped_by_id = user_id,
            master_object_id = self.mapped_indicator_id_1
        )

        self.mapped_indicator_with_data = locations[2].id
        indicator_map = SourceObjectMap.objects.create(
            source_object_code = 'Percent missed due to other reasons',
            content_type = 'indicator',
            mapped_by_id = user_id,
            master_object_id = self.mapped_indicator_with_data
        )

    def model_df_to_data(self,model_df,model):

        meta_ids = []

        non_null_df = model_df.where((notnull(model_df)), None)
        list_of_dicts = non_null_df.transpose().to_dict()

        for row_ix, row_dict in list_of_dicts.iteritems():

            row_id = model.objects.create(**row_dict)
            meta_ids.append(row_id)

        return meta_ids
<commit_msg>added test for uploading new data and overwriting old<commit_after>from django.test import TestCase
from django.contrib.auth.models import User
from django.conf import settings
from pandas import read_csv, notnull, to_datetime

from rhizome.etl_tasks.simple_upload_transform import SimpleDocTransform
from rhizome.models import *

class TransformUploadTestCase(TestCase):

    def __init__(self, *args, **kwargs):

        super(TransformUploadTestCase, self).__init__(*args, **kwargs)

    def setUp(self):

        self.create_metadata()
        self.user = User.objects.get(username = 'test')
        self.location_list = Location.objects.all().values_list('name',flat=True)

    def test_simple_transform(self):

        self.ingest_file('eoc_post_campaign.csv')

        the_value_from_the_database = DataPointComputed.objects.get(
                campaign_id = self.mapped_campaign_id,
                indicator_id = self.mapped_indicator_with_data,
                location_id = self.mapped_location_id
            ).value

        some_cell_value_from_the_file = 0.082670906
        ## find this from the data frame by selecting the cell where we have mapped the data..

        self.assertEqual(some_cell_value_from_the_file, the_value_from_the_database)

    def test_upload_new_data(self):

        file_and_cell_vals ={'eoc_post_campaign.csv': 0.082670906, 'modified_single_cell.csv': 0.0324}

        for file, cell_val_from_file in file_and_cell_vals.iteritems():
            self.ingest_file(file)

            the_value_from_the_database = DataPointComputed.objects.get(
                    campaign_id = self.mapped_campaign_id,
                    indicator_id = self.mapped_indicator_with_data,
                    location_id = self.mapped_location_id
                ).value

            #test that the file cell value reflects that in the database
            self.assertEqual(cell_val_from_file, the_value_from_the_database)

    def create_metadata(self):
        '''
        Creating the Indicator, location, Campaign, meta data needed for the
        system to aggregate / caclulate.
        '''

        top_lvl_tag = IndicatorTag.objects.create(id = 1, tag_name='Polio')

        campaign_df = read_csv('rhizome/tests/_data/campaigns.csv')
        campaign_df['top_lvl_indicator_tag_id'] = top_lvl_tag.id

        campaign_df['start_date'] = to_datetime(campaign_df['start_date'])
        campaign_df['end_date'] = to_datetime(campaign_df['end_date'])

        location_df= read_csv('rhizome/tests/_data/locations.csv')
        indicator_df = read_csv('rhizome/tests/_data/indicators.csv')

        office_id = Office.objects.create(id=1,name='test').id

        cache_job_id = CacheJob.objects.create(id = -2, \
            date_attempted = '2015-01-01',is_error = False)

        campaign_type = CampaignType.objects.create(id=1,name="test")

        locations = self.model_df_to_data(location_df,Location)
        campaigns = self.model_df_to_data(campaign_df,Campaign)
        indicators = self.model_df_to_data(indicator_df,Indicator)
        self.user_id = User.objects.create_user('test','test@test.com', 'test').id
        self.mapped_location_id = locations[0].id
        loc_map = SourceObjectMap.objects.create(
            source_object_code = 'AF001039003000000000',
            content_type = 'location',
            mapped_by_id = self.user_id,
            master_object_id = self.mapped_location_id
        )

        source_campaign_string = '2016 March NID OPV'
        self.mapped_campaign_id = campaigns[0].id
        campaign_map = SourceObjectMap.objects.create(
            source_object_code = source_campaign_string,
            content_type = 'campaign',
            mapped_by_id = self.user_id,
            master_object_id = self.mapped_campaign_id
        )
        self.mapped_indicator_id_0 = locations[0].id
        indicator_map = SourceObjectMap.objects.create(
            source_object_code = 'Percent missed children_PCA',
            content_type = 'indicator',
            mapped_by_id = self.user_id,
            master_object_id = self.mapped_indicator_id_0
        )

        self.mapped_indicator_id_1 = locations[1].id
        indicator_map = SourceObjectMap.objects.create(
            source_object_code = 'Percent missed due to not visited',
            content_type = 'indicator',
            mapped_by_id = self.user_id,
            master_object_id = self.mapped_indicator_id_1
        )

        self.mapped_indicator_with_data = locations[2].id
        indicator_map = SourceObjectMap.objects.create(
            source_object_code = 'Percent missed due to other reasons',
            content_type = 'indicator',
            mapped_by_id = self.user_id,
            master_object_id = self.mapped_indicator_with_data
        )

    def model_df_to_data(self,model_df,model):

        meta_ids = []

        non_null_df = model_df.where((notnull(model_df)), None)
        list_of_dicts = non_null_df.transpose().to_dict()

        for row_ix, row_dict in list_of_dicts.iteritems():

            row_id = model.objects.create(**row_dict)
            meta_ids.append(row_id)

        return meta_ids

    def ingest_file(self, file_name):
        ## create one doc ##
        document = Document.objects.create(
        doc_title = file_name,
        file_header = 'Campaign,Wardcode,uq_id,HHsampled,HHvisitedTEAMS,Marked0to59,UnImmun0to59,NOimmReas1,NOimmReas2,NOimmReas3,NOimmReas4,NOimmReas5,NOimmReas6,NOimmReas7,NOimmReas8,NOimmReas9,NOimmReas10,NOimmReas11,NOimmReas12,NOimmReas13,NOimmReas14,NOimmReas15,NOimmReas16,NOimmReas17,NOimmReas18,NOimmReas19,NOimmReas20,ZeroDose,TotalYoungest,YoungstRI,RAssessMrk,RCorctCAT,RIncorect,RXAssessMrk,RXCorctCAT,RXIncorect,STannounc,SRadio,STradlead,SReiliglead,SMosque,SNewspaper,SPoster,Sbanner,SRelative,SHworker,Scommmob,SNOTAWARE,Influence1,Influence2,Influence3,Influence4,Influence5,Influence6,Influence7,Influence8',
        created_by_id = self.user_id,
        guid = 'test')
        document.docfile = file_name
        document.save()
        sdt = SimpleDocTransform(self.user.id, document.id)
        sdt.main()

<|endoftext|>


<commit_before>
VERSION = '0.80.56'

<commit_msg>version bump for 0.80.57.<commit_after>
VERSION = '0.80.57'

<|endoftext|>