<commit_before># coding: utf-8
"""
JSON serialization and deserialization utilities.
"""

from __future__ import absolute_import, unicode_literals
import json
import datetime
import six
import inspect

try:
    from inspect import getfullargspec as getargspec
except ImportError:
    from inspect import getargspec

try:
    import numpy as np
except ImportError:
    np = None

try:
    import bson
except ImportError:
    bson = None

__author__ = "Shyue Ping Ong"
__copyright__ = "Copyright 2014, The Materials Virtual Lab"
__version__ = "0.1"
__maintainer__ = "Shyue Ping Ong"
__email__ = "ongsp@ucsd.edu"
__date__ = "1/24/14"

class MSONable(object):
    """
    This is a mix-in base class specifying an API for msonable objects. MSON
    is Monty JSON. Essentially, MSONable objects must implement an as_dict
    method, which must return a json serializable dict and must also support
    no arguments (though optional arguments to finetune the output is ok),
    and a from_dict class method that regenerates the object from the dict
    generated by the as_dict method. The as_dict method should contain the
    "@module" and "@class" keys which will allow the MontyEncoder to
    dynamically deserialize the class. E.g.::

        d["@module"] = self.__class__.__module__
        d["@module"] = self.__class__.__name__

    A default implementation is provided in MSONable, which automatically
    determines if the class already contains self.argname or self._argname
    attributes for every arg. If so, these will be used for serialization in
    the dict format. Similarly, the default from_dict will deserialization
    classes of such form. An example is given below::

        class MSONClass(MSONable):

        def __init__(self, a, b, c, d=1, **kwargs):
            self.a = a
            self.b = b
            self._c = c
            self._d = d
            self.kwargs = kwargs

    For such classes, you merely need to inherit from MSONable and you do not
    need to implement your own as_dict or from_dict protocol.
    """

    def as_dict(self):
        """
        A JSON serializable dict representation of an object.
        """
        d = {"@module": self.__class__.__module__,
             "@class": self.__class__.__name__}
        args = getargspec(self.__class__.__init__).args

        def rec_as_dict(obj):
            if isinstance(obj, list):
                return [rec_as_dict(it) for it in obj]
            elif isinstance(obj, dict):
                return {kk: rec_as_dict(vv) for kk, vv in obj.items()}
            elif hasattr(obj, "as_dict"):
                return obj.as_dict()
            return obj

        for c in args:
            if c != "self":
                try:
                    a = self.__getattribute__(c)

                except AttributeError:
                    try:
                        a = self.__getattribute__("_" + c)
                    except AttributeError:
                        raise NotImplementedError(
                            "Unable to automatically determine as_dict "
                            "format from class. MSONAble requires all "
                            "args to be present as either self.argname or "
                            "self._argname, and kwargs to be present under"
                            "a self.kwargs variable to automatically "
                            "determine the dict format. Alternatively, "
                            "you can implement both as_dict and from_dict.")
                d[c] = rec_as_dict(a)
        if hasattr(self, "kwargs"):
            d.update(**self.kwargs)
        if hasattr(self, "_kwargs"):
            d.update(**self._kwargs)
        return d

    @classmethod
    def from_dict(cls, d):
        decoded = {k: MontyDecoder().process_decoded(v) for k, v in d.items()
                   if not k.startswith("@")}
        return cls(**decoded)

    def to_json(self):
        """
        Returns a json string representation of the MSONable object.
        """
        return json.dumps(self, cls=MontyEncoder)

class MontyEncoder(json.JSONEncoder):
    """
    A Json Encoder which supports the MSONable API, plus adds support for
    numpy arrays, datetime objects, bson ObjectIds (requires bson).

    Usage::

        # Add it as a *cls* keyword when using json.dump
        json.dumps(object, cls=MontyEncoder)
    """

    def default(self, o):
        """
        Overriding default method for JSON encoding. This method does two
        things: (a) If an object has a to_dict property, return the to_dict
        output. (b) If the @module and @class keys are not in the to_dict,
        add them to the output automatically. If the object has no to_dict
        property, the default Python json encoder default method is called.

        Args:
            o: Python object.

        Return:
            Python dict representation.
        """
        if isinstance(o, datetime.datetime):
            return {"@module": "datetime", "@class": "datetime",
                    "string": o.__str__()}
        if np is not None:
            if isinstance(o, np.ndarray):
                return {"@module": "numpy",
                        "@class": "array",
                        "dtype": o.dtype.__str__(),
                        "data": o.tolist()}
            elif isinstance(o, np.generic):
                return o.item()
        if bson is not None:
            if isinstance(o, bson.objectid.ObjectId):
                return {"@module": "bson.objectid",
                        "@class": "ObjectId",
                        "oid": str(o)}

        try:
            d = o.as_dict()
            if "@module" not in d:
                d["@module"] = u"{}".format(o.__class__.__module__)
            if "@class" not in d:
                d["@class"] = u"{}".format(o.__class__.__name__)
            return d
        except AttributeError:
            return json.JSONEncoder.default(self, o)

class MontyDecoder(json.JSONDecoder):
    """
    A Json Decoder which supports the MSONable API. By default, the
    decoder attempts to find a module and name associated with a dict. If
    found, the decoder will generate a Pymatgen as a priority.  If that fails,
    the original decoded dictionary from the string is returned. Note that
    nested lists and dicts containing pymatgen object will be decoded correctly
    as well.

    Usage:

        # Add it as a *cls* keyword when using json.load
        json.loads(json_string, cls=MontyDecoder)
    """

    def process_decoded(self, d):
        """
        Recursive method to support decoding dicts and lists containing
        pymatgen objects.
        """
        if isinstance(d, dict):
            if "@module" in d and "@class" in d:
                modname = d["@module"]
                classname = d["@class"]
            else:
                modname = None
                classname = None
            if modname and modname not in ["bson.objectid", "numpy"]:
                if modname == "datetime" and classname == "datetime":
                    try:
                        dt = datetime.datetime.strptime(d["string"],
                                                        "%Y-%m-%d %H:%M:%S.%f")
                    except ValueError:
                        dt = datetime.datetime.strptime(d["string"],
                                                        "%Y-%m-%d %H:%M:%S")
                    return dt

                mod = __import__(modname, globals(), locals(), [classname], 0)
                if hasattr(mod, classname):
                    cls_ = getattr(mod, classname)
                    data = {k: v for k, v in d.items()
                            if k not in ["@module", "@class"]}
                    if hasattr(cls_, "from_dict"):
                        return cls_.from_dict(data)
            elif np is not None and modname == "numpy" and classname == \
                    "array":
                return np.array(d["data"], dtype=d["dtype"])

            elif (bson is not None) and modname == "bson.objectid" and \
                    classname == "ObjectId":
                return bson.objectid.ObjectId(d["oid"])

            return {self.process_decoded(k): self.process_decoded(v)
                    for k, v in d.items()}
        elif isinstance(d, list):
            return [self.process_decoded(x) for x in d]

        return d

    def decode(self, s):
        d = json.JSONDecoder.decode(self, s)
        return self.process_decoded(d)

class MSONError(Exception):
    """
    Exception class for serialization errors.
    """
    pass

def jsanitize(obj, strict=False, allow_bson=False):
    """
    This method cleans an input json-like object, either a list or a dict or
    some sequence, nested or otherwise, by converting all non-string
    dictionary keys (such as int and float) to strings, and also recursively
    encodes all objects using Monty's as_dict() protocol.

    Args:
        obj: input json-like object.
        strict (bool): This parameters sets the behavior when jsanitize
            encounters an object it does not understand. If strict is True,
            jsanitize will try to get the as_dict() attribute of the object. If
            no such attribute is found, an attribute error will be thrown. If
            strict is False, jsanitize will simply call str(object) to convert
            the object to a string representation.
        allow_bson (bool): This parameters sets the behavior when jsanitize
            encounters an bson supported type such as objectid and datetime. If
            True, such bson types will be ignored, allowing for proper
            insertion into MongoDb databases.

    Returns:
        Sanitized dict that can be json serialized.
    """
    if allow_bson and (isinstance(obj, (datetime.datetime, bytes)) or \
                       (bson is not None and isinstance(obj,
                                                        bson.objectid.ObjectId))):
        return obj
    if isinstance(obj, (list, tuple)):
        return [jsanitize(i, strict=strict, allow_bson=allow_bson) for i in obj]
    elif np is not None and isinstance(obj, np.ndarray):
        return [jsanitize(i, strict=strict, allow_bson=allow_bson) for i in
                obj.tolist()]
    elif isinstance(obj, dict):
        return {k.__str__(): jsanitize(v, strict=strict, allow_bson=allow_bson)
                for k, v in obj.items()}
    elif isinstance(obj, six.integer_types + tuple([float])):
        return obj
    elif obj is None:
        return None

    else:
        if not strict:
            return obj.__str__()
        else:
            if isinstance(obj, six.string_types):
                return obj.__str__()
            else:
                return jsanitize(obj.as_dict(), strict=strict,
                                 allow_bson=allow_bson)
<commit_msg>Renaming internal method to be more readable. Added support for tuples as well.<commit_after># coding: utf-8
"""
JSON serialization and deserialization utilities.
"""

from __future__ import absolute_import, unicode_literals
import json
import datetime
import six
import inspect

try:
    from inspect import getfullargspec as getargspec
except ImportError:
    from inspect import getargspec

try:
    import numpy as np
except ImportError:
    np = None

try:
    import bson
except ImportError:
    bson = None

__author__ = "Shyue Ping Ong"
__copyright__ = "Copyright 2014, The Materials Virtual Lab"
__version__ = "0.1"
__maintainer__ = "Shyue Ping Ong"
__email__ = "ongsp@ucsd.edu"
__date__ = "1/24/14"

class MSONable(object):
    """
    This is a mix-in base class specifying an API for msonable objects. MSON
    is Monty JSON. Essentially, MSONable objects must implement an as_dict
    method, which must return a json serializable dict and must also support
    no arguments (though optional arguments to finetune the output is ok),
    and a from_dict class method that regenerates the object from the dict
    generated by the as_dict method. The as_dict method should contain the
    "@module" and "@class" keys which will allow the MontyEncoder to
    dynamically deserialize the class. E.g.::

        d["@module"] = self.__class__.__module__
        d["@module"] = self.__class__.__name__

    A default implementation is provided in MSONable, which automatically
    determines if the class already contains self.argname or self._argname
    attributes for every arg. If so, these will be used for serialization in
    the dict format. Similarly, the default from_dict will deserialization
    classes of such form. An example is given below::

        class MSONClass(MSONable):

        def __init__(self, a, b, c, d=1, **kwargs):
            self.a = a
            self.b = b
            self._c = c
            self._d = d
            self.kwargs = kwargs

    For such classes, you merely need to inherit from MSONable and you do not
    need to implement your own as_dict or from_dict protocol.
    """

    def as_dict(self):
        """
        A JSON serializable dict representation of an object.
        """
        d = {"@module": self.__class__.__module__,
             "@class": self.__class__.__name__}
        args = getargspec(self.__class__.__init__).args

        def recursive_as_dict(obj):
            if isinstance(obj, (list, tuple)):
                return [recursive_as_dict(it) for it in obj]
            elif isinstance(obj, dict):
                return {kk: recursive_as_dict(vv) for kk, vv in obj.items()}
            elif hasattr(obj, "as_dict"):
                return obj.as_dict()
            return obj

        for c in args:
            if c != "self":
                try:
                    a = self.__getattribute__(c)

                except AttributeError:
                    try:
                        a = self.__getattribute__("_" + c)
                    except AttributeError:
                        raise NotImplementedError(
                            "Unable to automatically determine as_dict "
                            "format from class. MSONAble requires all "
                            "args to be present as either self.argname or "
                            "self._argname, and kwargs to be present under"
                            "a self.kwargs variable to automatically "
                            "determine the dict format. Alternatively, "
                            "you can implement both as_dict and from_dict.")
                d[c] = recursive_as_dict(a)
        if hasattr(self, "kwargs"):
            d.update(**self.kwargs)
        if hasattr(self, "_kwargs"):
            d.update(**self._kwargs)
        return d

    @classmethod
    def from_dict(cls, d):
        decoded = {k: MontyDecoder().process_decoded(v) for k, v in d.items()
                   if not k.startswith("@")}
        return cls(**decoded)

    def to_json(self):
        """
        Returns a json string representation of the MSONable object.
        """
        return json.dumps(self, cls=MontyEncoder)

class MontyEncoder(json.JSONEncoder):
    """
    A Json Encoder which supports the MSONable API, plus adds support for
    numpy arrays, datetime objects, bson ObjectIds (requires bson).

    Usage::

        # Add it as a *cls* keyword when using json.dump
        json.dumps(object, cls=MontyEncoder)
    """

    def default(self, o):
        """
        Overriding default method for JSON encoding. This method does two
        things: (a) If an object has a to_dict property, return the to_dict
        output. (b) If the @module and @class keys are not in the to_dict,
        add them to the output automatically. If the object has no to_dict
        property, the default Python json encoder default method is called.

        Args:
            o: Python object.

        Return:
            Python dict representation.
        """
        if isinstance(o, datetime.datetime):
            return {"@module": "datetime", "@class": "datetime",
                    "string": o.__str__()}
        if np is not None:
            if isinstance(o, np.ndarray):
                return {"@module": "numpy",
                        "@class": "array",
                        "dtype": o.dtype.__str__(),
                        "data": o.tolist()}
            elif isinstance(o, np.generic):
                return o.item()
        if bson is not None:
            if isinstance(o, bson.objectid.ObjectId):
                return {"@module": "bson.objectid",
                        "@class": "ObjectId",
                        "oid": str(o)}

        try:
            d = o.as_dict()
            if "@module" not in d:
                d["@module"] = u"{}".format(o.__class__.__module__)
            if "@class" not in d:
                d["@class"] = u"{}".format(o.__class__.__name__)
            return d
        except AttributeError:
            return json.JSONEncoder.default(self, o)

class MontyDecoder(json.JSONDecoder):
    """
    A Json Decoder which supports the MSONable API. By default, the
    decoder attempts to find a module and name associated with a dict. If
    found, the decoder will generate a Pymatgen as a priority.  If that fails,
    the original decoded dictionary from the string is returned. Note that
    nested lists and dicts containing pymatgen object will be decoded correctly
    as well.

    Usage:

        # Add it as a *cls* keyword when using json.load
        json.loads(json_string, cls=MontyDecoder)
    """

    def process_decoded(self, d):
        """
        Recursive method to support decoding dicts and lists containing
        pymatgen objects.
        """
        if isinstance(d, dict):
            if "@module" in d and "@class" in d:
                modname = d["@module"]
                classname = d["@class"]
            else:
                modname = None
                classname = None
            if modname and modname not in ["bson.objectid", "numpy"]:
                if modname == "datetime" and classname == "datetime":
                    try:
                        dt = datetime.datetime.strptime(d["string"],
                                                        "%Y-%m-%d %H:%M:%S.%f")
                    except ValueError:
                        dt = datetime.datetime.strptime(d["string"],
                                                        "%Y-%m-%d %H:%M:%S")
                    return dt

                mod = __import__(modname, globals(), locals(), [classname], 0)
                if hasattr(mod, classname):
                    cls_ = getattr(mod, classname)
                    data = {k: v for k, v in d.items()
                            if k not in ["@module", "@class"]}
                    if hasattr(cls_, "from_dict"):
                        return cls_.from_dict(data)
            elif np is not None and modname == "numpy" and classname == \
                    "array":
                return np.array(d["data"], dtype=d["dtype"])

            elif (bson is not None) and modname == "bson.objectid" and \
                    classname == "ObjectId":
                return bson.objectid.ObjectId(d["oid"])

            return {self.process_decoded(k): self.process_decoded(v)
                    for k, v in d.items()}
        elif isinstance(d, list):
            return [self.process_decoded(x) for x in d]

        return d

    def decode(self, s):
        d = json.JSONDecoder.decode(self, s)
        return self.process_decoded(d)

class MSONError(Exception):
    """
    Exception class for serialization errors.
    """
    pass

def jsanitize(obj, strict=False, allow_bson=False):
    """
    This method cleans an input json-like object, either a list or a dict or
    some sequence, nested or otherwise, by converting all non-string
    dictionary keys (such as int and float) to strings, and also recursively
    encodes all objects using Monty's as_dict() protocol.

    Args:
        obj: input json-like object.
        strict (bool): This parameters sets the behavior when jsanitize
            encounters an object it does not understand. If strict is True,
            jsanitize will try to get the as_dict() attribute of the object. If
            no such attribute is found, an attribute error will be thrown. If
            strict is False, jsanitize will simply call str(object) to convert
            the object to a string representation.
        allow_bson (bool): This parameters sets the behavior when jsanitize
            encounters an bson supported type such as objectid and datetime. If
            True, such bson types will be ignored, allowing for proper
            insertion into MongoDb databases.

    Returns:
        Sanitized dict that can be json serialized.
    """
    if allow_bson and (isinstance(obj, (datetime.datetime, bytes)) or \
                       (bson is not None and isinstance(obj,
                                                        bson.objectid.ObjectId))):
        return obj
    if isinstance(obj, (list, tuple)):
        return [jsanitize(i, strict=strict, allow_bson=allow_bson) for i in obj]
    elif np is not None and isinstance(obj, np.ndarray):
        return [jsanitize(i, strict=strict, allow_bson=allow_bson) for i in
                obj.tolist()]
    elif isinstance(obj, dict):
        return {k.__str__(): jsanitize(v, strict=strict, allow_bson=allow_bson)
                for k, v in obj.items()}
    elif isinstance(obj, six.integer_types + tuple([float])):
        return obj
    elif obj is None:
        return None

    else:
        if not strict:
            return obj.__str__()
        else:
            if isinstance(obj, six.string_types):
                return obj.__str__()
            else:
                return jsanitize(obj.as_dict(), strict=strict,
                                 allow_bson=allow_bson)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

import unittest
import os
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.alert import Alert
from selenium.webdriver.support.ui import Select, WebDriverWait
from test.testmodeltestcase import TestModelTestCase as Super

class QueryImportTestCase(Super):

    def setUp(self):
        Super.setUp(self)
        self.browser.get(self.base_url + '/importQueries.do')

    def import_query(self, xml):
        self.assertIn('Import Query', self.browser.title)
        self.elem('#xml').send_keys(xml)
        self.elem('#importQueriesForm input[type="submit"]').click()

    def assert_error_msg_contains(self, message):
        error_notification = self.wait_for_elem('#error_msg')
        self.wait().until(lambda d: error_notification.is_displayed())
        self.assertIn(message, error_notification.text)

    def test_bad_xml(self):
        bad_query = '<query model="testmodel" view="Employee.name Bank.name"/>'
        self.import_query(bad_query)
        self.assert_error_msg_contains('Multiple root classes')

    def test_empty_form(self):
        self.import_query('')
        self.assert_error_msg_contains('end of file')

    def test_from_file(self):
        file_input = self.elem('#file')
        file_input.send_keys(os.path.abspath('data/employee-query.xml'))

        self.elem('#importQueriesForm input[type="submit"]').click()
        self.assertEquals('Employee', self.elem('.typeSelected').text)
        self.assertEquals(3, len(self.browser.find_elements_by_class_name('viewpath')))

    def test_bad_file(self):
        file_input = self.elem('#file')
        file_input.send_keys(os.path.abspath('data/bad-query.xml'))
        self.elem('#importQueriesForm input[type="submit"]').click()

        self.assert_error_msg_contains('Multiple root classes')

<commit_msg>Tightened up importing<commit_after># -*- coding: utf-8 -*-

import unittest
import os
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.alert import Alert
from selenium.webdriver.support.ui import Select, WebDriverWait
from test.testmodeltestcase import TestModelTestCase as Super

class QueryImportTestCase(Super):

    def setUp(self):
        Super.setUp(self)
        self.browser.get(self.base_url + '/importQueries.do')

    def import_query(self, xml):
        self.assertIn('Import Query', self.browser.title)
        self.wait_for_elem('#xml').send_keys(xml)
        self.elem('#importQueriesForm input[type="submit"]').click()

    def assert_error_msg_contains(self, message):
        error_notification = self.wait_for_elem('#error_msg')
        self.wait().until(lambda d: error_notification.is_displayed())
        self.assertIn(message, error_notification.text)

    def test_bad_xml(self):
        bad_query = '<query model="testmodel" view="Employee.name Bank.name"/>'
        self.import_query(bad_query)
        self.assert_error_msg_contains('Multiple root classes')

    def test_empty_form(self):
        self.import_query('')
        self.assert_error_msg_contains('end of file')

    def test_from_form(self):
        with open('data/employee-query.xml') as f:
            xml = f.read()
            self.import_query(xml)

        self.assertEquals('Employee', self.wait_for_elem('.typeSelected').text)
        self.assertEquals(3, len(self.browser.find_elements_by_class_name('viewpath')))

    def test_from_file(self):
        file_input = self.elem('#file')
        file_input.send_keys(os.path.abspath('data/employee-query.xml'))
        self.elem('#importQueriesForm input[type="submit"]').click()

        self.assertEquals('Employee', self.wait_for_elem('.typeSelected').text)
        self.assertEquals(3, len(self.browser.find_elements_by_class_name('viewpath')))

    def test_bad_file(self):
        file_input = self.elem('#file')
        file_input.send_keys(os.path.abspath('data/bad-query.xml'))
        self.elem('#importQueriesForm input[type="submit"]').click()

        self.assert_error_msg_contains('Multiple root classes')

<|endoftext|>


<commit_before># Copyright (c) 2018, Compiler Explorer Authors
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
#     * Redistributions of source code must retain the above copyright notice,
#       this list of conditions and the following disclaimer.
#     * Redistributions in binary form must reproduce the above copyright
#       notice, this list of conditions and the following disclaimer in the
#       documentation and/or other materials provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.

import subprocess

def format_commit(url, commit):
    splitted_commit = commit.split(' * ')
    print(splitted_commit)
    return '
\n' \
           '
\n' \
           '        {}\n' \
           '            \n' \
           '        \n' \
           '
\n' \
           '
\n' \
           '
{}
\n' \
           '
\n' \
           '
\n'.format(url, splitted_commit[0], splitted_commit[0], splitted_commit[1])

def get_commits(repo):
    coms = subprocess.check_output(['git', 'log', '--date=local', '--after="3 months ago"', '--grep=^* ', '--oneline'])
    with open('static/changelog.html', 'w') as f:
        for commit in coms.splitlines():
            f.write(format_commit(repo, commit))

if __name__ == '__main__':
    get_commits('https://github.com/mattgodbolt/compiler-explorer/')
<commit_msg>Rename confusing variable<commit_after># Copyright (c) 2018, Compiler Explorer Authors
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
#     * Redistributions of source code must retain the above copyright notice,
#       this list of conditions and the following disclaimer.
#     * Redistributions in binary form must reproduce the above copyright
#       notice, this list of conditions and the following disclaimer in the
#       documentation and/or other materials provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.

import subprocess

def format_commit(url, commit):
    grouped_commit = commit.split(' * ')
    print(grouped_commit)
    return '
\n' \
           '
\n' \
           '        {}\n' \
           '            \n' \
           '        \n' \
           '
\n' \
           '
\n' \
           '
{}
\n' \
           '
\n' \
           '
\n'.format(url, grouped_commit[0], grouped_commit[0], grouped_commit[1])

def get_commits(repo):
    coms = subprocess.check_output(['git', 'log', '--date=local', '--after="3 months ago"', '--grep=^* ', '--oneline'])
    with open('static/changelog.html', 'w') as f:
        for commit in coms.splitlines():
            f.write(format_commit(repo, commit))

if __name__ == '__main__':
    get_commits('https://github.com/mattgodbolt/compiler-explorer/')
<|endoftext|>


<commit_before>#!/usr/bin/env python
# coding: utf-8
# TST-Online Update
# (C) 2012 Dalton Serey
# This module contains helper functions and classes for TST scripts.

from __future__ import print_function
from __future__ import unicode_literals

import os
import sys
import json
import codecs
import signal

from subprocess import Popen, PIPE, CalledProcessError

# optionally import requests
try:
    import requests
    we_have_requests = True
except ImportError:
    we_have_requests = False

# Constants
TSTDIR = os.path.expanduser("~/.tst/")
TSTCONFIG = os.path.expanduser(TSTDIR + "config.json")
TSTJSON = os.path.expanduser("./tst.json")

def date_handler(obj):
    if hasattr(obj, 'isoformat'):
        return obj.isoformat()
    elif hasattr(obj, 'email'):
        return obj.email()

    return obj

def data2json(data):
    return json.dumps(
        data,
        default=date_handler,
        indent=2,
        separators=(',', ': '),
        ensure_ascii=False)

def to_unicode(obj, encoding='utf-8'):
    assert isinstance(obj, basestring), type(obj)
    if isinstance(obj, unicode):
        return obj

    for encoding in ['utf-8', 'latin1']:
        try:
            obj = unicode(obj, encoding)
            return obj
        except UnicodeDecodeError:
            pass

    assert False, "tst: non-recognized encoding"

class CorruptedConfigFile(Exception): pass
class ConnectionFail(Exception): pass

class TSTOnline:

    def __init__(self, token):
        self.token = token

    class Response:

        def json(self):
            return json.loads(self.text)

    def get(self, url, headers={}):
        curl_command = ['curl', '-v', '-s']
        headers['Authorization'] = 'Bearer %s' % self.token
        for hname, hvalue in headers.items():
            curl_command.append('-H')
            curl_command.append('%s: %s' % (hname, hvalue))

        curl_command.append(url)

        signal.alarm(20000) # timeout in seconds
        process = Popen(curl_command, stdout=PIPE, stderr=PIPE)
        try:
            stdout, stderr = map(to_unicode, process.communicate())
            signal.alarm(0) # reset alarm for future use...
            process.wait()
        except CutTimeOut: # timeout!!!
            process.terminate()
            raise ConnectionFail()

        # raw data
        response = self.Response()
        response.stderr = stderr
        response.stdout = stdout

        # curl messages
        lines = [l[2:] for l in stderr.splitlines() if l and l[0] == '*']
        response.curl_messages = "\n".join(lines)

        # request headers
        lines = [l[2:] for l in stderr.splitlines() if l and l[0] == '>']
        response.request_headers = "\n".join(lines)

        # response headers
        lines = [l[2:] for l in stderr.splitlines() if l and l[0] == '<']
        response.headers = "\n".join(lines)

        if not response.headers:
            raise ConnectionFail("can't connect to tst online")

        # text
        if response.headers[0:4] == "HTTP":
            status_line = response.headers.splitlines()[0]
            response.status_code = int(status_line.split()[1])
        else:
            response.status_code = None

        if response.status_code:
            response.text = stdout
        else:
            response.text = None

        return response

    def patch(self, url, payload):
        curl_command = ['curl', '-X', 'PATCH', '-v', '-s']
        headers = {}
        headers['Authorization'] = 'Bearer %s' % self.token
        for hname, hvalue in headers.items():
            curl_command.append('-H')
            curl_command.append('%s: %s' % (hname, hvalue))

        curl_command.append(url)
        curl_command.append('-d')
        data = "%s" % json.dumps(payload)
        curl_command.append(data)

        signal.alarm(20000) # timeout in seconds
        process = Popen(curl_command, stdout=PIPE, stderr=PIPE)
        try:
            stdout, stderr = map(to_unicode, process.communicate())
            signal.alarm(0) # reset alarm for future use...
            process.wait()
        except CutTimeOut: # timeout!!!
            process.terminate()
            raise ConnectionFail()

        # raw data
        response = self.Response()
        response.stderr = stderr
        response.stdout = stdout

        # curl messages
        lines = [l[2:] for l in stderr.splitlines() if l and l[0] == '*']
        response.curl_messages = "\n".join(lines)

        # request headers
        lines = [l[2:] for l in stderr.splitlines() if l and l[0] == '>']
        response.request_headers = "\n".join(lines)

        # response headers
        lines = [l[2:] for l in stderr.splitlines() if l and l[0] == '<']
        response.headers = "\n".join(lines)

        if not response.headers:
            raise ConnectionFail("can't connect to tst online")

        # text
        if response.headers[0:4] == "HTTP":
            status_line = response.headers.splitlines()[0]
            response.status_code = int(status_line.split()[1])
        else:
            response.status_code = None

        if response.status_code:
            response.text = stdout
        else:
            response.text = None

        return response

def read_json(jsonfile, exit_on_fail=False):

    if not os.path.exists(jsonfile):
        return {}

    try:
        with codecs.open(jsonfile, mode='r', encoding='utf-8') as f:
            tstjson = json.loads(to_unicode(f.read()))

    except ValueError:
        msg = "tst: %s is corrupted" % jsonfile
        if exit_on_fail:
            print(msg, file=sys.stderr)
            sys.exit()

        raise CorruptedConfigFile(msg)

    return tstjson

def read_tstjson(exit=False):

    if not os.path.exists(TSTJSON):
        return {}

    try:
        with codecs.open(TSTJSON, mode='r', encoding='utf-8') as f:
            tstjson = json.loads(to_unicode(f.read()))

    except ValueError:
        msg = "tst: %s is corrupted" % TSTJSON
        if exit:
            print(msg, file=sys.stderr)
            sys.exit()

        raise CorruptedConfigFile(msg)

    return tstjson

def read_config(exit=False):

    # create config file if it doesn't exist
    if os.path.exists(TSTCONFIG):
        try:
            with codecs.open(TSTCONFIG, mode='r', encoding='utf-8') as f:
                config = json.loads(to_unicode(f.read()))

        except ValueError:
            msg = "tst: %s is corrupted" % TSTCONFIG
            if exit:
                print(msg, file=sys.stderr)
                sys.exit()

            raise CorruptedConfigFile(msg)

    else:
        if not os.path.exists(TSTDIR):
            os.mkdir(TSTDIR)

        config = {
            'url': 'http://tst-online.appspot.com',
            'cookies': {}
        }
        save_config(config)

    return config

def save_config(config):
    with codecs.open(TSTCONFIG, mode="w", encoding='utf-8') as f:
        f.write(json.dumps(
            config,
            indent=2,
            separators=(',', ': ')
        ))

def save_json(jsondata, jsonfile):
    with codecs.open(jsonfile, mode="w", encoding='utf-8') as f:
        f.write(json.dumps(
            jsondata,
            indent=2,
            separators=(',', ': ')
        ))

def save_tstjson(tstjson):
    with codecs.open(TSTJSON, mode="w", encoding='utf-8') as f:
        f.write(json.dumps(
            tstjson,
            indent=2,
            separators=(',', ': ')
        ))

def requests_required(method):
    def check_requests(self, *args):
        if not we_have_requests:
          print("tst: this command requires requests")

        return method(self, *args)

    return check_requests

@requests_required
def get(url):
    session = requests.session()
    resp = session.get(url, cookies=config['cookies'])
    return resp.text

@requests_required
def post(url, data):
    url = config['baseurl'] + url
    session = requests.session()
    resp = session.post(url, cookies=config['cookies'], data=json.dumps(data))
    return resp.text

if __name__ == "__main__":
    print("tstlib is a helper module for tst commands")
<commit_msg>Make commit add tst version in http headers<commit_after>#!/usr/bin/env python
# coding: utf-8
# TST-Online Update
# (C) 2012 Dalton Serey
# This module contains helper functions and classes for TST scripts.

from __future__ import print_function
from __future__ import unicode_literals

import os
import sys
import json
import codecs
import signal

from subprocess import Popen, PIPE, CalledProcessError

# optionally import requests
try:
    import requests
    we_have_requests = True
except ImportError:
    we_have_requests = False

# Constants
TSTDIR = os.path.expanduser("~/.tst/")
TSTCONFIG = os.path.expanduser(TSTDIR + "config.json")
TSTJSON = os.path.expanduser("./tst.json")

def date_handler(obj):
    if hasattr(obj, 'isoformat'):
        return obj.isoformat()
    elif hasattr(obj, 'email'):
        return obj.email()

    return obj

def data2json(data):
    return json.dumps(
        data,
        default=date_handler,
        indent=2,
        separators=(',', ': '),
        ensure_ascii=False)

def to_unicode(obj, encoding='utf-8'):
    assert isinstance(obj, basestring), type(obj)
    if isinstance(obj, unicode):
        return obj

    for encoding in ['utf-8', 'latin1']:
        try:
            obj = unicode(obj, encoding)
            return obj
        except UnicodeDecodeError:
            pass

    assert False, "tst: non-recognized encoding"

class CorruptedConfigFile(Exception): pass
class ConnectionFail(Exception): pass

class TSTOnline:

    def __init__(self, token):
        self.token = token

    class Response:

        def json(self):
            return json.loads(self.text)

    def get(self, url, headers={}):
        curl_command = ['curl', '-v', '-s']
        headers['Authorization'] = 'Bearer %s' % self.token
        for hname, hvalue in headers.items():
            curl_command.append('-H')
            curl_command.append('%s: %s' % (hname, hvalue))

        curl_command.append(url)

        signal.alarm(20000) # timeout in seconds
        process = Popen(curl_command, stdout=PIPE, stderr=PIPE)
        try:
            stdout, stderr = map(to_unicode, process.communicate())
            signal.alarm(0) # reset alarm for future use...
            process.wait()
        except CutTimeOut: # timeout!!!
            process.terminate()
            raise ConnectionFail()

        # raw data
        response = self.Response()
        response.stderr = stderr
        response.stdout = stdout

        # curl messages
        lines = [l[2:] for l in stderr.splitlines() if l and l[0] == '*']
        response.curl_messages = "\n".join(lines)

        # request headers
        lines = [l[2:] for l in stderr.splitlines() if l and l[0] == '>']
        response.request_headers = "\n".join(lines)

        # response headers
        lines = [l[2:] for l in stderr.splitlines() if l and l[0] == '<']
        response.headers = "\n".join(lines)

        if not response.headers:
            raise ConnectionFail("can't connect to tst online")

        # text
        if response.headers[0:4] == "HTTP":
            status_line = response.headers.splitlines()[0]
            response.status_code = int(status_line.split()[1])
        else:
            response.status_code = None

        if response.status_code:
            response.text = stdout
        else:
            response.text = None

        return response

    def patch(self, url, payload):
        curl_command = ['curl', '-X', 'PATCH', '-v', '-s']
        headers = {}
        headers['Authorization'] = 'Bearer %s' % self.token
        config = read_config()
        headers['TST-CLI-Release'] = config.get('release')
        for hname, hvalue in headers.items():
            curl_command.append('-H')
            curl_command.append('%s: %s' % (hname, hvalue))

        curl_command.append(url)
        curl_command.append('-d')
        data = "%s" % json.dumps(payload)
        curl_command.append(data)

        signal.alarm(20000) # timeout in seconds
        process = Popen(curl_command, stdout=PIPE, stderr=PIPE)
        try:
            stdout, stderr = map(to_unicode, process.communicate())
            signal.alarm(0) # reset alarm for future use...
            process.wait()
        except CutTimeOut: # timeout!!!
            process.terminate()
            raise ConnectionFail()

        # raw data
        response = self.Response()
        response.stderr = stderr
        response.stdout = stdout

        # curl messages
        lines = [l[2:] for l in stderr.splitlines() if l and l[0] == '*']
        response.curl_messages = "\n".join(lines)

        # request headers
        lines = [l[2:] for l in stderr.splitlines() if l and l[0] == '>']
        response.request_headers = "\n".join(lines)

        # response headers
        lines = [l[2:] for l in stderr.splitlines() if l and l[0] == '<']
        response.headers = "\n".join(lines)

        if not response.headers:
            raise ConnectionFail("can't connect to tst online")

        # text
        if response.headers[0:4] == "HTTP":
            status_line = response.headers.splitlines()[0]
            response.status_code = int(status_line.split()[1])
        else:
            response.status_code = None

        if response.status_code:
            response.text = stdout
        else:
            response.text = None

        return response

def read_json(jsonfile, exit_on_fail=False):

    if not os.path.exists(jsonfile):
        return {}

    try:
        with codecs.open(jsonfile, mode='r', encoding='utf-8') as f:
            tstjson = json.loads(to_unicode(f.read()))

    except ValueError:
        msg = "tst: %s is corrupted" % jsonfile
        if exit_on_fail:
            print(msg, file=sys.stderr)
            sys.exit()

        raise CorruptedConfigFile(msg)

    return tstjson

def read_tstjson(exit=False):

    if not os.path.exists(TSTJSON):
        return {}

    try:
        with codecs.open(TSTJSON, mode='r', encoding='utf-8') as f:
            tstjson = json.loads(to_unicode(f.read()))

    except ValueError:
        msg = "tst: %s is corrupted" % TSTJSON
        if exit:
            print(msg, file=sys.stderr)
            sys.exit()

        raise CorruptedConfigFile(msg)

    return tstjson

def read_config(exit=False):

    # create config file if it doesn't exist
    if os.path.exists(TSTCONFIG):
        try:
            with codecs.open(TSTCONFIG, mode='r', encoding='utf-8') as f:
                config = json.loads(to_unicode(f.read()))

        except ValueError:
            msg = "tst: %s is corrupted" % TSTCONFIG
            if exit:
                print(msg, file=sys.stderr)
                sys.exit()

            raise CorruptedConfigFile(msg)

    else:
        if not os.path.exists(TSTDIR):
            os.mkdir(TSTDIR)

        config = {
            'url': 'http://tst-online.appspot.com',
            'cookies': {}
        }
        save_config(config)

    return config

def save_config(config):
    with codecs.open(TSTCONFIG, mode="w", encoding='utf-8') as f:
        f.write(json.dumps(
            config,
            indent=2,
            separators=(',', ': ')
        ))

def save_json(jsondata, jsonfile):
    with codecs.open(jsonfile, mode="w", encoding='utf-8') as f:
        f.write(json.dumps(
            jsondata,
            indent=2,
            separators=(',', ': ')
        ))

def save_tstjson(tstjson):
    with codecs.open(TSTJSON, mode="w", encoding='utf-8') as f:
        f.write(json.dumps(
            tstjson,
            indent=2,
            separators=(',', ': ')
        ))

def requests_required(method):
    def check_requests(self, *args):
        if not we_have_requests:
          print("tst: this command requires requests")

        return method(self, *args)

    return check_requests

@requests_required
def get(url):
    session = requests.session()
    resp = session.get(url, cookies=config['cookies'])
    return resp.text

@requests_required
def post(url, data):
    url = config['baseurl'] + url
    session = requests.session()
    resp = session.post(url, cookies=config['cookies'], data=json.dumps(data))
    return resp.text

if __name__ == "__main__":
    print("tstlib is a helper module for tst commands")
<|endoftext|>


<commit_before>#!/usr/bin/env python
"""Tests for HDF5 stuff"""
import tempfile
from os.path import join, dirname

import numpy as np
import tables as tb

from km3pipe import Blob, Module, Pipeline, Pump
from km3pipe.dataclasses import Table
from km3pipe.io import HDF5Pump, HDF5Sink   # noqa
from km3pipe.tools import insert_prefix_to_dtype
from km3pipe.testing import TestCase

DATA_DIR = join(dirname(__file__), '../../kp-data/test_data/')

class TestMultiTable(TestCase):
    def setUp(self):
        self.foo = np.array([
            (1.0, 2.0, 3.0),
            (4.0, 5.0, 6.0),
        ], dtype=[('a', '<f8'), ('b', '<f8'), ('c', '<f8'), ])
        self.bar = np.array([
            (10.0, 20.0, 30.0),
            (40.0, 50.0, 60.0),
        ], dtype=[('aa', '<f8'), ('bb', '<f8'), ('cc', '<f8'), ])
        self.tabs = {'foo': self.foo, 'bar': self.bar}
        self.where = '/lala'
        self.fobj = tempfile.NamedTemporaryFile(delete=True)
        self.h5name = self.fobj.name
        self.h5file = tb.open_file(
            # create the file in memory only
            self.h5name, 'w', driver="H5FD_CORE", driver_core_backing_store=0)
        for name, tab in self.tabs.items():
            self.h5file.create_table(self.where, name=name, obj=tab,
                                     createparents=True)

    def tearDown(self):
        self.h5file.close()
        self.fobj.close()

    def test_name_insert(self):
        exp_foo = ('foo_a', 'foo_b', 'foo_c')
        exp_bar = ('bar_aa', 'bar_bb', 'bar_cc')
        pref_foo = insert_prefix_to_dtype(self.tabs['foo'], 'foo')
        pref_bar = insert_prefix_to_dtype(self.tabs['bar'], 'bar')
        self.assertEqual(exp_foo, pref_foo.dtype.names)
        self.assertEqual(exp_bar, pref_bar.dtype.names)

    # def test_group_read(self):
    #    tabs = read_group(self.h5file.root)
    #    exp_cols = (
    #        'bar_aa', 'bar_bb', 'bar_cc',
    #        'foo_a', 'foo_b', 'foo_c',
    #    )
    #    exp_shape = (2, 6)
    #    res_shape = tabs.shape
    #    res_cols = tuple(tabs.columns)
    #    print(exp_cols)
    #    print(res_cols)
    #    self.assertEqual(exp_shape, res_shape)
    #    self.assertEqual(exp_cols, res_cols)

class TestH5Pump(TestCase):
    def setUp(self):
        self.fname = join(DATA_DIR,  'numu_cc_test.h5')

    def test_init_sets_filename_if_no_keyword_arg_is_passed(self):
        p = HDF5Pump(self.fname)
        self.assertEqual(self.fname, p.filename)
        p.finish()

    def test_context(self):
        with HDF5Pump(self.fname) as h5:
            self.assertEqual(self.fname, h5.filename)
            assert h5[0] is not None
            for blob in h5:
                assert blob is not None
                break

    def test_standalone(self):
        pump = HDF5Pump(filename=self.fname)
        next(pump)
        pump.finish()

    def test_pipe(self):
        p = Pipeline()
        p.attach(HDF5Pump, filename=self.fname)
        p.drain()

    def test_event_info_is_not_empty(self):
        self.fname = join(DATA_DIR,  'test_event_info.h5')
        class Printer(Module):
            def process(self, blob):
                assert blob['EventInfo'].size != 0
                return blob
        p = Pipeline()
        p.attach(HDF5Pump, filename=self.fname)
        p.attach(Printer)
        p.drain()

class TestH5Sink(TestCase):
    def setUp(self):
        self.fname = join(DATA_DIR, 'numu_cc_test.h5')
        self.fobj = tempfile.NamedTemporaryFile(delete=True)
        self.out = tb.open_file(self.fobj.name, "w", driver="H5FD_CORE",
                                driver_core_backing_store=0)

    def tearDown(self):
        self.out.close()
        self.fobj.close()

    # def test_init_has_to_be_explicit(self):
    #     with self.assertRaises(TypeError):
    #         HDF5Sink(self.out)

    def test_pipe(self):
        p = Pipeline()
        p.attach(HDF5Pump, filename=self.fname)
        p.attach(HDF5Sink, h5file=self.out)
        p.drain()

    def test_scalars(self):
        out = tb.open_file('foobar_scalar', "a", driver="H5FD_CORE",
                           driver_core_backing_store=0)

        def pu(blob):
            return {'foo': 42.0}

        p = Pipeline()
        p.attach(pu)
        p.attach(HDF5Sink, h5file=out, keep_open=True)
        p.drain(2)
        node = out.root.misc
        assert node is not None
        assert node.cols is not None
        assert 'foo' in set(node.cols._v_colnames)
        out.close()

class TestH5SinkConsistency(TestCase):
    def test_h5_consistency_for_tables_without_group_id(self):
        fobj = tempfile.NamedTemporaryFile(delete=True)
        fname = fobj.name

        class DummyPump(Pump):
            def configure(self):
                self.count = 0

            def process(self, blob):
                self.count += 10
                tab = Table({'a': self.count, 'b': 1}, h5loc='tab')
                return Blob({'tab': tab})

        pipe = Pipeline()
        pipe.attach(DummyPump)
        pipe.attach(HDF5Sink, filename=fname)
        pipe.drain(5)

        with tb.File(fname) as f:
            a = f.get_node("/tab")[:]['a']
            b = f.get_node("/tab")[:]['b']
            group_id = f.get_node("/tab")[:]['group_id']
        assert np.allclose([10, 20, 30, 40, 50], a)
        assert np.allclose([1, 1, 1, 1, 1], b)
        assert np.allclose([0, 1, 2, 3, 4], group_id)
        fobj.close()

    def test_h5_consistency_for_tables_without_group_id_and_multiple_keys(self):
        fobj = tempfile.NamedTemporaryFile(delete=True)
        fname = fobj.name

        class DummyPump(Pump):
            def configure(self):
                self.count = 0

            def process(self, blob):
                self.count += 10
                tab1 = Table({'a': self.count, 'b': 1}, h5loc='tab1')
                tab2 = Table({'c': self.count + 1, 'd': 2}, h5loc='tab2')
                return Blob({'tab1': tab1, 'tab2': tab2})

        pipe = Pipeline()
        pipe.attach(DummyPump)
        pipe.attach(HDF5Sink, filename=fname)
        pipe.drain(5)

        with tb.File(fname) as f:
            a = f.get_node("/tab1")[:]['a']
            b = f.get_node("/tab1")[:]['b']
            c = f.get_node("/tab2")[:]['c']
            d = f.get_node("/tab2")[:]['d']
            group_id_1 = f.get_node("/tab1")[:]['group_id']
            group_id_2 = f.get_node("/tab1")[:]['group_id']
        assert np.allclose([10, 20, 30, 40, 50], a)
        assert np.allclose([1, 1, 1, 1, 1], b)
        assert np.allclose([0, 1, 2, 3, 4], group_id_1)
        assert np.allclose([11, 21, 31, 41, 51], c)
        assert np.allclose([2, 2, 2, 2, 2], d)
        assert np.allclose([0, 1, 2, 3, 4], group_id_2)
        fobj.close()

    def test_h5_consistency_for_tables_with_custom_group_id(self):
        fobj = tempfile.NamedTemporaryFile(delete=True)
        fname = fobj.name

        class DummyPump(Pump):
            def process(self, blob):
                tab = Table({'group_id': 2}, h5loc='tab')
                return Blob({'tab': tab})

        pipe = Pipeline()
        pipe.attach(DummyPump)
        pipe.attach(HDF5Sink, filename=fname)
        pipe.drain(5)

        with tb.File(fname) as f:
            group_id = f.get_node("/tab")[:]['group_id']

        assert np.allclose([2, 2, 2, 2, 2], group_id)

        fobj.close()

class TestHDF5PumpConsistency(TestCase):
    def test_hdf5_readout(self):
        fobj = tempfile.NamedTemporaryFile(delete=True)
        fname = fobj.name

        class DummyPump(Pump):
            def configure(self):
                self.count = 0

            def process(self, blob):
                self.count += 1
                tab = Table({'a': self.count * 10, 'b': 1}, h5loc='tab')
                tab2 = Table({'a': np.arange(self.count)}, h5loc='tab2')
                blob['Tab'] = tab
                blob['Tab2'] = tab2
                return blob

        pipe = Pipeline()
        pipe.attach(DummyPump)
        pipe.attach(HDF5Sink, filename=fname)
        pipe.drain(5)

        class BlobTester(Module):

            def configure(self):
                self.index = 0

            def process(self, blob):
                self.index += 1
                assert 'GroupInfo' in blob
                assert 'Tab' in blob
                print(self.index)
                print(blob['Tab'])
                print(blob['Tab']['a'])
                assert self.index - 1 == blob['GroupInfo'].group_id
                assert self.index * 10 == blob['Tab']['a']
                assert 1 == blob['Tab']['b'] == 1
                assert np.allclose(np.arange(self.index), blob['Tab2']['a'])
                return blob

        pipe = Pipeline()
        pipe.attach(HDF5Pump, filename=fname)
        pipe.attach(BlobTester)
        pipe.drain()

        fobj.close()

    def test_hdf5_readout_split_tables(self):
        fobj = tempfile.NamedTemporaryFile(delete=True)
        fname = fobj.name

        class DummyPump(Pump):
            def configure(self):
                self.count = 0

            def process(self, blob):
                self.count += 1
                tab = Table({'a': self.count * 10, 'b': 1},
                            h5loc='/tab', split_h5=True)
                blob['Tab'] = tab
                return blob

        pipe = Pipeline()
        pipe.attach(DummyPump)
        pipe.attach(HDF5Sink, filename=fname)
        pipe.drain(5)

        class BlobTester(Module):

            def configure(self):
                self.index = 0

            def process(self, blob):
                self.index += 1
                assert 'GroupInfo' in blob
                assert 'Tab' in blob
                assert self.index - 1 == blob['GroupInfo'].group_id
                assert self.index * 10 == blob['Tab']['a']
                assert 1 == blob['Tab']['b'] == 1
                return blob

        pipe = Pipeline()
        pipe.attach(HDF5Pump, filename=fname)
        pipe.attach(BlobTester)
        pipe.drain()

        fobj.close()
<commit_msg>Add additional test to check event_info readin<commit_after>#!/usr/bin/env python
"""Tests for HDF5 stuff"""
import tempfile
from os.path import join, dirname

import numpy as np
import tables as tb

from km3pipe import Blob, Module, Pipeline, Pump
from km3pipe.dataclasses import Table
from km3pipe.io import HDF5Pump, HDF5Sink   # noqa
from km3pipe.tools import insert_prefix_to_dtype
from km3pipe.testing import TestCase

DATA_DIR = join(dirname(__file__), '../../kp-data/test_data/')

class TestMultiTable(TestCase):
    def setUp(self):
        self.foo = np.array([
            (1.0, 2.0, 3.0),
            (4.0, 5.0, 6.0),
        ], dtype=[('a', '<f8'), ('b', '<f8'), ('c', '<f8'), ])
        self.bar = np.array([
            (10.0, 20.0, 30.0),
            (40.0, 50.0, 60.0),
        ], dtype=[('aa', '<f8'), ('bb', '<f8'), ('cc', '<f8'), ])
        self.tabs = {'foo': self.foo, 'bar': self.bar}
        self.where = '/lala'
        self.fobj = tempfile.NamedTemporaryFile(delete=True)
        self.h5name = self.fobj.name
        self.h5file = tb.open_file(
            # create the file in memory only
            self.h5name, 'w', driver="H5FD_CORE", driver_core_backing_store=0)
        for name, tab in self.tabs.items():
            self.h5file.create_table(self.where, name=name, obj=tab,
                                     createparents=True)

    def tearDown(self):
        self.h5file.close()
        self.fobj.close()

    def test_name_insert(self):
        exp_foo = ('foo_a', 'foo_b', 'foo_c')
        exp_bar = ('bar_aa', 'bar_bb', 'bar_cc')
        pref_foo = insert_prefix_to_dtype(self.tabs['foo'], 'foo')
        pref_bar = insert_prefix_to_dtype(self.tabs['bar'], 'bar')
        self.assertEqual(exp_foo, pref_foo.dtype.names)
        self.assertEqual(exp_bar, pref_bar.dtype.names)

    # def test_group_read(self):
    #    tabs = read_group(self.h5file.root)
    #    exp_cols = (
    #        'bar_aa', 'bar_bb', 'bar_cc',
    #        'foo_a', 'foo_b', 'foo_c',
    #    )
    #    exp_shape = (2, 6)
    #    res_shape = tabs.shape
    #    res_cols = tuple(tabs.columns)
    #    print(exp_cols)
    #    print(res_cols)
    #    self.assertEqual(exp_shape, res_shape)
    #    self.assertEqual(exp_cols, res_cols)

class TestH5Pump(TestCase):
    def setUp(self):
        self.fname = join(DATA_DIR,  'numu_cc_test.h5')

    def test_init_sets_filename_if_no_keyword_arg_is_passed(self):
        p = HDF5Pump(self.fname)
        self.assertEqual(self.fname, p.filename)
        p.finish()

    def test_context(self):
        with HDF5Pump(self.fname) as h5:
            self.assertEqual(self.fname, h5.filename)
            assert h5[0] is not None
            for blob in h5:
                assert blob is not None
                break

    def test_standalone(self):
        pump = HDF5Pump(filename=self.fname)
        next(pump)
        pump.finish()

    def test_pipe(self):
        p = Pipeline()
        p.attach(HDF5Pump, filename=self.fname)
        p.drain()

    def test_event_info_is_not_empty(self):
        self.fname = join(DATA_DIR,  'test_event_info.h5')
        class Printer(Module):
            def process(self, blob):
                assert blob['EventInfo'].size != 0
                return blob
        p = Pipeline()
        p.attach(HDF5Pump, filename=self.fname)
        p.attach(Printer)
        p.drain()

    def test_event_info_has_correct_group_id(self):
        self.fname = join(DATA_DIR,  'test_event_info.h5')

        class Printer(Module):
            def configure(self):
                self.index = 0

            def process(self, blob):
                assert blob['EventInfo'][0].group_id == self.index
                self.index += 1
                return blob

        p = Pipeline()
        p.attach(HDF5Pump, filename=self.fname)
        p.attach(Printer)
        p.drain()

class TestH5Sink(TestCase):
    def setUp(self):
        self.fname = join(DATA_DIR, 'numu_cc_test.h5')
        self.fobj = tempfile.NamedTemporaryFile(delete=True)
        self.out = tb.open_file(self.fobj.name, "w", driver="H5FD_CORE",
                                driver_core_backing_store=0)

    def tearDown(self):
        self.out.close()
        self.fobj.close()

    # def test_init_has_to_be_explicit(self):
    #     with self.assertRaises(TypeError):
    #         HDF5Sink(self.out)

    def test_pipe(self):
        p = Pipeline()
        p.attach(HDF5Pump, filename=self.fname)
        p.attach(HDF5Sink, h5file=self.out)
        p.drain()

    def test_scalars(self):
        out = tb.open_file('foobar_scalar', "a", driver="H5FD_CORE",
                           driver_core_backing_store=0)

        def pu(blob):
            return {'foo': 42.0}

        p = Pipeline()
        p.attach(pu)
        p.attach(HDF5Sink, h5file=out, keep_open=True)
        p.drain(2)
        node = out.root.misc
        assert node is not None
        assert node.cols is not None
        assert 'foo' in set(node.cols._v_colnames)
        out.close()

class TestH5SinkConsistency(TestCase):
    def test_h5_consistency_for_tables_without_group_id(self):
        fobj = tempfile.NamedTemporaryFile(delete=True)
        fname = fobj.name

        class DummyPump(Pump):
            def configure(self):
                self.count = 0

            def process(self, blob):
                self.count += 10
                tab = Table({'a': self.count, 'b': 1}, h5loc='tab')
                return Blob({'tab': tab})

        pipe = Pipeline()
        pipe.attach(DummyPump)
        pipe.attach(HDF5Sink, filename=fname)
        pipe.drain(5)

        with tb.File(fname) as f:
            a = f.get_node("/tab")[:]['a']
            b = f.get_node("/tab")[:]['b']
            group_id = f.get_node("/tab")[:]['group_id']
        assert np.allclose([10, 20, 30, 40, 50], a)
        assert np.allclose([1, 1, 1, 1, 1], b)
        assert np.allclose([0, 1, 2, 3, 4], group_id)
        fobj.close()

    def test_h5_consistency_for_tables_without_group_id_and_multiple_keys(self):
        fobj = tempfile.NamedTemporaryFile(delete=True)
        fname = fobj.name

        class DummyPump(Pump):
            def configure(self):
                self.count = 0

            def process(self, blob):
                self.count += 10
                tab1 = Table({'a': self.count, 'b': 1}, h5loc='tab1')
                tab2 = Table({'c': self.count + 1, 'd': 2}, h5loc='tab2')
                return Blob({'tab1': tab1, 'tab2': tab2})

        pipe = Pipeline()
        pipe.attach(DummyPump)
        pipe.attach(HDF5Sink, filename=fname)
        pipe.drain(5)

        with tb.File(fname) as f:
            a = f.get_node("/tab1")[:]['a']
            b = f.get_node("/tab1")[:]['b']
            c = f.get_node("/tab2")[:]['c']
            d = f.get_node("/tab2")[:]['d']
            group_id_1 = f.get_node("/tab1")[:]['group_id']
            group_id_2 = f.get_node("/tab1")[:]['group_id']
        assert np.allclose([10, 20, 30, 40, 50], a)
        assert np.allclose([1, 1, 1, 1, 1], b)
        assert np.allclose([0, 1, 2, 3, 4], group_id_1)
        assert np.allclose([11, 21, 31, 41, 51], c)
        assert np.allclose([2, 2, 2, 2, 2], d)
        assert np.allclose([0, 1, 2, 3, 4], group_id_2)
        fobj.close()

    def test_h5_consistency_for_tables_with_custom_group_id(self):
        fobj = tempfile.NamedTemporaryFile(delete=True)
        fname = fobj.name

        class DummyPump(Pump):
            def process(self, blob):
                tab = Table({'group_id': 2}, h5loc='tab')
                return Blob({'tab': tab})

        pipe = Pipeline()
        pipe.attach(DummyPump)
        pipe.attach(HDF5Sink, filename=fname)
        pipe.drain(5)

        with tb.File(fname) as f:
            group_id = f.get_node("/tab")[:]['group_id']

        assert np.allclose([2, 2, 2, 2, 2], group_id)

        fobj.close()

class TestHDF5PumpConsistency(TestCase):
    def test_hdf5_readout(self):
        fobj = tempfile.NamedTemporaryFile(delete=True)
        fname = fobj.name

        class DummyPump(Pump):
            def configure(self):
                self.count = 0

            def process(self, blob):
                self.count += 1
                tab = Table({'a': self.count * 10, 'b': 1}, h5loc='tab')
                tab2 = Table({'a': np.arange(self.count)}, h5loc='tab2')
                blob['Tab'] = tab
                blob['Tab2'] = tab2
                return blob

        pipe = Pipeline()
        pipe.attach(DummyPump)
        pipe.attach(HDF5Sink, filename=fname)
        pipe.drain(5)

        class BlobTester(Module):

            def configure(self):
                self.index = 0

            def process(self, blob):
                self.index += 1
                assert 'GroupInfo' in blob
                assert 'Tab' in blob
                print(self.index)
                print(blob['Tab'])
                print(blob['Tab']['a'])
                assert self.index - 1 == blob['GroupInfo'].group_id
                assert self.index * 10 == blob['Tab']['a']
                assert 1 == blob['Tab']['b'] == 1
                assert np.allclose(np.arange(self.index), blob['Tab2']['a'])
                return blob

        pipe = Pipeline()
        pipe.attach(HDF5Pump, filename=fname)
        pipe.attach(BlobTester)
        pipe.drain()

        fobj.close()

    def test_hdf5_readout_split_tables(self):
        fobj = tempfile.NamedTemporaryFile(delete=True)
        fname = fobj.name

        class DummyPump(Pump):
            def configure(self):
                self.count = 0

            def process(self, blob):
                self.count += 1
                tab = Table({'a': self.count * 10, 'b': 1},
                            h5loc='/tab', split_h5=True)
                blob['Tab'] = tab
                return blob

        pipe = Pipeline()
        pipe.attach(DummyPump)
        pipe.attach(HDF5Sink, filename=fname)
        pipe.drain(5)

        class BlobTester(Module):

            def configure(self):
                self.index = 0

            def process(self, blob):
                self.index += 1
                assert 'GroupInfo' in blob
                assert 'Tab' in blob
                assert self.index - 1 == blob['GroupInfo'].group_id
                assert self.index * 10 == blob['Tab']['a']
                assert 1 == blob['Tab']['b'] == 1
                return blob

        pipe = Pipeline()
        pipe.attach(HDF5Pump, filename=fname)
        pipe.attach(BlobTester)
        pipe.drain()

        fobj.close()
<|endoftext|>


<commit_before># (C) Datadog, Inc. 2020-present
# All rights reserved
# Licensed under a 3-clause BSD style license (see LICENSE)
from six.moves.urllib.parse import urlparse

from datadog_checks.base import ConfigurationError, OpenMetricsBaseCheck

from .metrics import ADDITIONAL_METRICS_MAP, INSTANCE_DEFAULT_METRICS

class ScyllaCheck(OpenMetricsBaseCheck):
    """
    Collect Scylla metrics from Prometheus endpoint
    """

    DEFAULT_METRIC_LIMIT = 0

    def __init__(self, name, init_config, instances):

        instance = instances[0]

        endpoint = instance.get('prometheus_url')
        if endpoint is None:
            raise ConfigurationError("Unable to find prometheus URL in config file.")

        # extract additional metrics requested and validate the correct names
        metric_groups = instance.get('metric_groups', [])
        additional_metrics = []
        if metric_groups:
            errors = []
            for group in metric_groups:
                try:
                    additional_metrics.append(ADDITIONAL_METRICS_MAP[group])
                except KeyError:
                    errors.append(group)

            if errors:
                raise ConfigurationError(
                    'Invalid metric_groups found in scylla conf.yaml: {}'.format(', '.join(errors))
                )

        metrics = INSTANCE_DEFAULT_METRICS + additional_metrics

        tags = instance.get('tags', [])

        # include hostname:port for server tag
        tags.append('server:{}'.format(urlparse(endpoint).netloc))

        instance.update(
            {
                'prometheus_url': endpoint,
                'namespace': 'scylla',
                'metrics': metrics,
                'tags': tags,
                'metadata_metric_name': 'scylla_scylladb_current_version',
                'metadata_label_map': {'version': 'version'},
                'send_histogram_buckets': True,
            }
        )

        super(ScyllaCheck, self).__init__(name, init_config, [instance])
<commit_msg>Fix typo in explicit send_histograms_buckets option (#5867)<commit_after># (C) Datadog, Inc. 2020-present
# All rights reserved
# Licensed under a 3-clause BSD style license (see LICENSE)
from six.moves.urllib.parse import urlparse

from datadog_checks.base import ConfigurationError, OpenMetricsBaseCheck

from .metrics import ADDITIONAL_METRICS_MAP, INSTANCE_DEFAULT_METRICS

class ScyllaCheck(OpenMetricsBaseCheck):
    """
    Collect Scylla metrics from Prometheus endpoint
    """

    DEFAULT_METRIC_LIMIT = 0

    def __init__(self, name, init_config, instances):

        instance = instances[0]

        endpoint = instance.get('prometheus_url')
        if endpoint is None:
            raise ConfigurationError("Unable to find prometheus URL in config file.")

        # extract additional metrics requested and validate the correct names
        metric_groups = instance.get('metric_groups', [])
        additional_metrics = []
        if metric_groups:
            errors = []
            for group in metric_groups:
                try:
                    additional_metrics.append(ADDITIONAL_METRICS_MAP[group])
                except KeyError:
                    errors.append(group)

            if errors:
                raise ConfigurationError(
                    'Invalid metric_groups found in scylla conf.yaml: {}'.format(', '.join(errors))
                )

        metrics = INSTANCE_DEFAULT_METRICS + additional_metrics

        tags = instance.get('tags', [])

        # include hostname:port for server tag
        tags.append('server:{}'.format(urlparse(endpoint).netloc))

        instance.update(
            {
                'prometheus_url': endpoint,
                'namespace': 'scylla',
                'metrics': metrics,
                'tags': tags,
                'metadata_metric_name': 'scylla_scylladb_current_version',
                'metadata_label_map': {'version': 'version'},
                'send_histograms_buckets': True,  # Default, but ensures we collect histograms sent by Scylla.
            }
        )

        super(ScyllaCheck, self).__init__(name, init_config, [instance])
<|endoftext|>


<commit_before># encoding: utf-8

"""
.. codeauthor:: Tsuyoshi Hombashi <gogogo.vm@gmail.com>
"""

from __future__ import absolute_import
import math

from ._align_getter import align_getter
from ._container import MinMaxContainer
from ._error import TypeConversionError
from ._interface import DataPeropertyInterface
from .type import Typecode
from .type import FloatTypeChecker

from ._function import is_nan
from ._function import get_number_of_digit
from ._function import get_text_len

from ._factory import NoneTypeFactory
from ._factory import StringTypeFactory
from ._factory import IntegerTypeFactory
from ._factory import FloatTypeFactory
from ._factory import BoolTypeFactory
from ._factory import DateTimeTypeFactory
from ._factory import InfinityTypeFactory
from ._factory import NanTypeFactory

def default_bool_converter(value):
    return value

def default_datetime_converter(value):
    return value

class DataPeropertyBase(DataPeropertyInterface):
    __slots__ = ("__datetime_format_str")

    @property
    def format_str(self):
        format_str = {
            Typecode.NONE: "",
            Typecode.INT: "d",
            Typecode.BOOL: "",
            Typecode.DATETIME: self.__datetime_format_str,
        }.get(self.typecode)

        if format_str is not None:
            return format_str

        if self.typecode in (Typecode.FLOAT, Typecode.INFINITY, Typecode.NAN):
            if is_nan(self.decimal_places):
                return "f"

            return ".%df" % (self.decimal_places)

        return "s"

    def __init__(self, datetime_format_str):
        self.__datetime_format_str = datetime_format_str

class DataProperty(DataPeropertyBase):
    __slots__ = (
        "__data",
        "__typecode",
        "__align",
        "__integer_digits",
        "__decimal_places",
        "__additional_format_len",
        "__str_len",
    )

    __type_factory_list = [
        NoneTypeFactory(),
        IntegerTypeFactory(),
        InfinityTypeFactory(),
        NanTypeFactory(),
        FloatTypeFactory(),
        BoolTypeFactory(),
        DateTimeTypeFactory(),
        StringTypeFactory(),
    ]

    @property
    def align(self):
        return self.__align

    @property
    def decimal_places(self):
        """
        :return:
            Decimal places if the ``data`` is ``float``.
            Returns ``0`` if the ``data`` is ``int``.
            Otherwise, returns ``float("nan")``.
        :rtype: int
        """

        return self.__decimal_places

    @property
    def typecode(self):
        """
        Return the type code that corresponds to the type of the ``data``.

        :return:
            One of the constants that are defined in the ``Typecode`` class.
        :rtype: int
        """

        return self.__typecode

    @property
    def data(self):
        """
        :return: Original data.
        :rtype: Original data type.
        """

        return self.__data

    @property
    def str_len(self):
        """
        :return: Length of the ``data`` as a string.
        :rtype: int
        """

        return self.__str_len

    @property
    def integer_digits(self):
        """
        :return:
            Integer digits if the ``data`` is ``int``/``float``.
            Otherwise, returns ``float("nan")``.
        :rtype: int
        """

        return self.__integer_digits

    @property
    def additional_format_len(self):
        return self.__additional_format_len

    def __init__(
            self, data,
            none_value=None, inf_value=float("inf"), nan_value=float("nan"),
            bool_converter=default_bool_converter,
            datetime_converter=default_datetime_converter,
            datetime_format_str="%Y-%m-%dT%H:%M:%S%z",
            is_convert=True,
            replace_tabs_with_spaces=True, tab_length=2):
        super(DataProperty, self).__init__(datetime_format_str)

        self.__set_data(data, none_value, inf_value, nan_value, is_convert)
        self.__convert_data(bool_converter, datetime_converter)
        self.__replace_tabs(replace_tabs_with_spaces, tab_length)
        self.__align = align_getter.get_align_from_typecode(self.typecode)

        try:
            integer_digits, decimal_places = get_number_of_digit(data)
        except OverflowError:
            integer_digits = float("nan")
            decimal_places = float("nan")
        self.__integer_digits = integer_digits
        self.__decimal_places = decimal_places
        self.__additional_format_len = self.__get_additional_format_len()
        self.__str_len = self.__get_str_len()

    def __repr__(self):
        element_list = []

        if self.typecode == Typecode.DATETIME:
            element_list.append(
                "data={:s}".format(str(self.data)))
        else:
            element_list.append(
                ("data={:" + self.format_str + "}").format(self.data))

        element_list.extend([
            "typename=" + Typecode.get_typename(self.typecode),
            "align=" + str(self.align),
            "str_len=" + str(self.str_len),
            "integer_digits=" + str(self.integer_digits),
            "decimal_places=" + str(self.decimal_places),
            "additional_format_len=" + str(self.additional_format_len),
        ])

        return ", ".join(element_list)

    def __get_additional_format_len(self):
        if not FloatTypeChecker(self.data).is_type():
            return 0

        format_len = 0

        if float(self.data) < 0:
            # for minus character
            format_len += 1

        return format_len

    def __get_base_float_len(self):
        if any([self.integer_digits < 0, self.decimal_places < 0]):
            raise ValueError()

        float_len = self.integer_digits + self.decimal_places
        if self.decimal_places > 0:
            # for dot
            float_len += 1

        return float_len

    def __get_str_len(self):
        if self.typecode == Typecode.INT:
            return self.integer_digits + self.additional_format_len

        if self.typecode == Typecode.FLOAT:
            return self.__get_base_float_len() + self.additional_format_len

        if self.typecode == Typecode.DATETIME:
            full_format_str = "{:" + self.format_str + "}"
            return len(full_format_str.format(self.data))

        return get_text_len(self.data)

    def __set_data(
            self, data, none_value, inf_value, nan_value, is_convert):
        special_value_table = {
            Typecode.NONE: none_value,
            Typecode.INFINITY: inf_value,
            Typecode.NAN: nan_value,
        }

        for type_factory in self.__type_factory_list:
            checker = type_factory.type_checker_factory.create(
                data, is_convert)
            if not checker.is_type():
                continue

            self.__typecode = checker.typecode

            special_value = special_value_table.get(self.__typecode)
            if special_value is not None:
                self.__data = special_value
                return

            self.__data = type_factory.value_converter_factory.create(
                data).convert()

            return

        raise TypeConversionError(
            "failed to convert: " + Typecode.get_typename(self.__typecode))

    def __convert_data(self, bool_converter, datetime_converter):
        if self.typecode == Typecode.BOOL:
            self.__data = bool_converter(self.__data)
        elif self.typecode == Typecode.DATETIME:
            self.__data = datetime_converter(self.__data)

    def __replace_tabs(self, replace_tabs_with_spaces, tab_length):
        if not replace_tabs_with_spaces:
            return

        try:
            self.__data = self.__data.replace("\t", " " * tab_length)
        except (TypeError, AttributeError):
            pass

class ColumnDataProperty(DataPeropertyBase):
    __slots__ = (
        "__typecode_bitmap",
        "__str_len",
        "__minmax_integer_digits",
        "__minmax_decimal_places",
        "__minmax_additional_format_len",
        "__data_prop_list",
    )

    __FACTORY_TABLE = {
        Typecode.NONE: NoneTypeFactory(),
        Typecode.STRING: StringTypeFactory(),
        Typecode.INT: IntegerTypeFactory(),
        Typecode.INFINITY: InfinityTypeFactory(),
        Typecode.NAN: NanTypeFactory(),
        Typecode.FLOAT: FloatTypeFactory(),
        Typecode.BOOL: BoolTypeFactory(),
        Typecode.DATETIME: DateTimeTypeFactory(),
    }

    @property
    def align(self):
        return align_getter.get_align_from_typecode(self.typecode)

    @property
    def decimal_places(self):
        try:
            avg = self.minmax_decimal_places.mean()
        except TypeError:
            return float("nan")

        if is_nan(avg):
            return float("nan")

        return int(math.ceil(avg))

    @property
    def typecode(self):
        return self.__get_typecode_from_bitmap()

    @property
    def padding_len(self):
        if self.typecode != Typecode.FLOAT:
            return self.__str_len

        max_len = self.__str_len
        float_format = "%" + self.format_str

        for data_prop in self.__data_prop_list:
            if data_prop.typecode in [Typecode.INFINITY, Typecode.NAN]:
                continue

            try:
                format_str = float_format % (data_prop.data)
            except TypeError:
                continue

            max_len = max(max_len, len(format_str))

        return max_len

    @property
    def minmax_integer_digits(self):
        return self.__minmax_integer_digits

    @property
    def minmax_decimal_places(self):
        return self.__minmax_decimal_places

    @property
    def minmax_additional_format_len(self):
        return self.__minmax_additional_format_len

    @property
    def type_factory(self):
        return self.__FACTORY_TABLE.get(self.typecode)

    def __init__(
            self,
            min_padding_len=0,
            datetime_format_str="%Y-%m-%dT%H:%M:%S%z"):
        super(ColumnDataProperty, self).__init__(datetime_format_str)

        self.__typecode_bitmap = Typecode.NONE
        self.__str_len = min_padding_len
        self.__minmax_integer_digits = MinMaxContainer()
        self.__minmax_decimal_places = MinMaxContainer()
        self.__minmax_additional_format_len = MinMaxContainer()
        self.__data_prop_list = []

    def __repr__(self):
        return ", ".join([
            "typename=" + Typecode.get_typename(self.typecode),
            "align=" + str(self.align),
            "padding_len=" + str(self.padding_len),
            "integer_digits=(%s)" % (str(self.minmax_integer_digits)),
            "decimal_places=(%s)" % (str(self.minmax_decimal_places)),
            "additional_format_len=(%s)" % (
                str(self.minmax_additional_format_len)),
        ])

    def update_header(self, dataprop):
        self.__update(dataprop)

    def update_body(self, dataprop):
        self.__typecode_bitmap |= dataprop.typecode
        self.__update(dataprop)
        self.__data_prop_list.append(dataprop)

    def __is_not_single_typecode(self, typecode):
        return all([
            self.__typecode_bitmap & typecode,
            self.__typecode_bitmap & ~typecode,
        ])

    def __is_float_typecode(self):
        number_typecode = (
            Typecode.INT | Typecode.FLOAT | Typecode.INFINITY | Typecode.NAN)

        if self.__is_not_single_typecode(number_typecode):
            return False

        if bin(self.__typecode_bitmap & number_typecode).count("1") >= 2:
            return True

        return False

    def __get_typecode_from_bitmap(self):
        if self.__is_float_typecode():
            return Typecode.FLOAT

        if any([
            self.__is_not_single_typecode(Typecode.BOOL),
            self.__is_not_single_typecode(Typecode.DATETIME),
        ]):
            return Typecode.STRING

        typecode_list = [
            Typecode.STRING,
            Typecode.FLOAT,
            Typecode.INT,
            Typecode.DATETIME,
            Typecode.BOOL,
            Typecode.INFINITY,
            Typecode.NAN,
        ]

        for typecode in typecode_list:
            if self.__typecode_bitmap & typecode:
                return typecode

        if self.__typecode_bitmap == Typecode.NONE:
            return Typecode.NONE

        return Typecode.STRING

    def __update(self, dataprop):
        self.__str_len = max(self.__str_len, dataprop.str_len)

        if dataprop.typecode in (Typecode.FLOAT, Typecode.INT):
            self.__minmax_integer_digits.update(dataprop.integer_digits)
            self.__minmax_decimal_places.update(dataprop.decimal_places)

        self.__minmax_additional_format_len.update(
            dataprop.additional_format_len)
<commit_msg>Change string formatter to format method<commit_after># encoding: utf-8

"""
.. codeauthor:: Tsuyoshi Hombashi <gogogo.vm@gmail.com>
"""

from __future__ import absolute_import
import math

from ._align_getter import align_getter
from ._container import MinMaxContainer
from ._error import TypeConversionError
from ._interface import DataPeropertyInterface
from .type import Typecode
from .type import FloatTypeChecker

from ._function import is_nan
from ._function import get_number_of_digit
from ._function import get_text_len

from ._factory import NoneTypeFactory
from ._factory import StringTypeFactory
from ._factory import IntegerTypeFactory
from ._factory import FloatTypeFactory
from ._factory import BoolTypeFactory
from ._factory import DateTimeTypeFactory
from ._factory import InfinityTypeFactory
from ._factory import NanTypeFactory

def default_bool_converter(value):
    return value

def default_datetime_converter(value):
    return value

class DataPeropertyBase(DataPeropertyInterface):
    __slots__ = ("__datetime_format_str")

    @property
    def format_str(self):
        format_str = {
            Typecode.NONE: "",
            Typecode.INT: "d",
            Typecode.BOOL: "",
            Typecode.DATETIME: self.__datetime_format_str,
        }.get(self.typecode)

        if format_str is not None:
            return format_str

        if self.typecode in (Typecode.FLOAT, Typecode.INFINITY, Typecode.NAN):
            if is_nan(self.decimal_places):
                return "f"

            return ".%df" % (self.decimal_places)

        return "s"

    def __init__(self, datetime_format_str):
        self.__datetime_format_str = datetime_format_str

class DataProperty(DataPeropertyBase):
    __slots__ = (
        "__data",
        "__typecode",
        "__align",
        "__integer_digits",
        "__decimal_places",
        "__additional_format_len",
        "__str_len",
    )

    __type_factory_list = [
        NoneTypeFactory(),
        IntegerTypeFactory(),
        InfinityTypeFactory(),
        NanTypeFactory(),
        FloatTypeFactory(),
        BoolTypeFactory(),
        DateTimeTypeFactory(),
        StringTypeFactory(),
    ]

    @property
    def align(self):
        return self.__align

    @property
    def decimal_places(self):
        """
        :return:
            Decimal places if the ``data`` is ``float``.
            Returns ``0`` if the ``data`` is ``int``.
            Otherwise, returns ``float("nan")``.
        :rtype: int
        """

        return self.__decimal_places

    @property
    def typecode(self):
        """
        Return the type code that corresponds to the type of the ``data``.

        :return:
            One of the constants that are defined in the ``Typecode`` class.
        :rtype: int
        """

        return self.__typecode

    @property
    def data(self):
        """
        :return: Original data.
        :rtype: Original data type.
        """

        return self.__data

    @property
    def str_len(self):
        """
        :return: Length of the ``data`` as a string.
        :rtype: int
        """

        return self.__str_len

    @property
    def integer_digits(self):
        """
        :return:
            Integer digits if the ``data`` is ``int``/``float``.
            Otherwise, returns ``float("nan")``.
        :rtype: int
        """

        return self.__integer_digits

    @property
    def additional_format_len(self):
        return self.__additional_format_len

    def __init__(
            self, data,
            none_value=None, inf_value=float("inf"), nan_value=float("nan"),
            bool_converter=default_bool_converter,
            datetime_converter=default_datetime_converter,
            datetime_format_str="%Y-%m-%dT%H:%M:%S%z",
            is_convert=True,
            replace_tabs_with_spaces=True, tab_length=2):
        super(DataProperty, self).__init__(datetime_format_str)

        self.__set_data(data, none_value, inf_value, nan_value, is_convert)
        self.__convert_data(bool_converter, datetime_converter)
        self.__replace_tabs(replace_tabs_with_spaces, tab_length)
        self.__align = align_getter.get_align_from_typecode(self.typecode)

        try:
            integer_digits, decimal_places = get_number_of_digit(data)
        except OverflowError:
            integer_digits = float("nan")
            decimal_places = float("nan")
        self.__integer_digits = integer_digits
        self.__decimal_places = decimal_places
        self.__additional_format_len = self.__get_additional_format_len()
        self.__str_len = self.__get_str_len()

    def __repr__(self):
        element_list = []

        if self.typecode == Typecode.DATETIME:
            element_list.append(
                "data={:s}".format(str(self.data)))
        else:
            element_list.append(
                ("data={:" + self.format_str + "}").format(self.data))

        element_list.extend([
            "typename=" + Typecode.get_typename(self.typecode),
            "align=" + str(self.align),
            "str_len=" + str(self.str_len),
            "integer_digits=" + str(self.integer_digits),
            "decimal_places=" + str(self.decimal_places),
            "additional_format_len=" + str(self.additional_format_len),
        ])

        return ", ".join(element_list)

    def __get_additional_format_len(self):
        if not FloatTypeChecker(self.data).is_type():
            return 0

        format_len = 0

        if float(self.data) < 0:
            # for minus character
            format_len += 1

        return format_len

    def __get_base_float_len(self):
        if any([self.integer_digits < 0, self.decimal_places < 0]):
            raise ValueError()

        float_len = self.integer_digits + self.decimal_places
        if self.decimal_places > 0:
            # for dot
            float_len += 1

        return float_len

    def __get_str_len(self):
        if self.typecode == Typecode.INT:
            return self.integer_digits + self.additional_format_len

        if self.typecode == Typecode.FLOAT:
            return self.__get_base_float_len() + self.additional_format_len

        if self.typecode == Typecode.DATETIME:
            full_format_str = "{:" + self.format_str + "}"
            return len(full_format_str.format(self.data))

        return get_text_len(self.data)

    def __set_data(
            self, data, none_value, inf_value, nan_value, is_convert):
        special_value_table = {
            Typecode.NONE: none_value,
            Typecode.INFINITY: inf_value,
            Typecode.NAN: nan_value,
        }

        for type_factory in self.__type_factory_list:
            checker = type_factory.type_checker_factory.create(
                data, is_convert)
            if not checker.is_type():
                continue

            self.__typecode = checker.typecode

            special_value = special_value_table.get(self.__typecode)
            if special_value is not None:
                self.__data = special_value
                return

            self.__data = type_factory.value_converter_factory.create(
                data).convert()

            return

        raise TypeConversionError(
            "failed to convert: " + Typecode.get_typename(self.__typecode))

    def __convert_data(self, bool_converter, datetime_converter):
        if self.typecode == Typecode.BOOL:
            self.__data = bool_converter(self.__data)
        elif self.typecode == Typecode.DATETIME:
            self.__data = datetime_converter(self.__data)

    def __replace_tabs(self, replace_tabs_with_spaces, tab_length):
        if not replace_tabs_with_spaces:
            return

        try:
            self.__data = self.__data.replace("\t", " " * tab_length)
        except (TypeError, AttributeError):
            pass

class ColumnDataProperty(DataPeropertyBase):
    __slots__ = (
        "__typecode_bitmap",
        "__str_len",
        "__minmax_integer_digits",
        "__minmax_decimal_places",
        "__minmax_additional_format_len",
        "__data_prop_list",
    )

    __FACTORY_TABLE = {
        Typecode.NONE: NoneTypeFactory(),
        Typecode.STRING: StringTypeFactory(),
        Typecode.INT: IntegerTypeFactory(),
        Typecode.INFINITY: InfinityTypeFactory(),
        Typecode.NAN: NanTypeFactory(),
        Typecode.FLOAT: FloatTypeFactory(),
        Typecode.BOOL: BoolTypeFactory(),
        Typecode.DATETIME: DateTimeTypeFactory(),
    }

    @property
    def align(self):
        return align_getter.get_align_from_typecode(self.typecode)

    @property
    def decimal_places(self):
        try:
            avg = self.minmax_decimal_places.mean()
        except TypeError:
            return float("nan")

        if is_nan(avg):
            return float("nan")

        return int(math.ceil(avg))

    @property
    def typecode(self):
        return self.__get_typecode_from_bitmap()

    @property
    def padding_len(self):
        if self.typecode != Typecode.FLOAT:
            return self.__str_len

        max_len = self.__str_len
        full_format_str = "{:" + self.format_str + "}"

        for data_prop in self.__data_prop_list:
            if data_prop.typecode in [Typecode.INFINITY, Typecode.NAN]:
                continue

            try:
                formatted_value = full_format_str.format(data_prop.data)
            except ValueError:
                continue

            max_len = max(max_len, len(formatted_value))

        return max_len

    @property
    def minmax_integer_digits(self):
        return self.__minmax_integer_digits

    @property
    def minmax_decimal_places(self):
        return self.__minmax_decimal_places

    @property
    def minmax_additional_format_len(self):
        return self.__minmax_additional_format_len

    @property
    def type_factory(self):
        return self.__FACTORY_TABLE.get(self.typecode)

    def __init__(
            self,
            min_padding_len=0,
            datetime_format_str="%Y-%m-%dT%H:%M:%S%z"):
        super(ColumnDataProperty, self).__init__(datetime_format_str)

        self.__typecode_bitmap = Typecode.NONE
        self.__str_len = min_padding_len
        self.__minmax_integer_digits = MinMaxContainer()
        self.__minmax_decimal_places = MinMaxContainer()
        self.__minmax_additional_format_len = MinMaxContainer()
        self.__data_prop_list = []

    def __repr__(self):
        return ", ".join([
            "typename=" + Typecode.get_typename(self.typecode),
            "align=" + str(self.align),
            "padding_len=" + str(self.padding_len),
            "integer_digits=(%s)" % (str(self.minmax_integer_digits)),
            "decimal_places=(%s)" % (str(self.minmax_decimal_places)),
            "additional_format_len=(%s)" % (
                str(self.minmax_additional_format_len)),
        ])

    def update_header(self, dataprop):
        self.__update(dataprop)

    def update_body(self, dataprop):
        self.__typecode_bitmap |= dataprop.typecode
        self.__update(dataprop)
        self.__data_prop_list.append(dataprop)

    def __is_not_single_typecode(self, typecode):
        return all([
            self.__typecode_bitmap & typecode,
            self.__typecode_bitmap & ~typecode,
        ])

    def __is_float_typecode(self):
        number_typecode = (
            Typecode.INT | Typecode.FLOAT | Typecode.INFINITY | Typecode.NAN)

        if self.__is_not_single_typecode(number_typecode):
            return False

        if bin(self.__typecode_bitmap & number_typecode).count("1") >= 2:
            return True

        return False

    def __get_typecode_from_bitmap(self):
        if self.__is_float_typecode():
            return Typecode.FLOAT

        if any([
            self.__is_not_single_typecode(Typecode.BOOL),
            self.__is_not_single_typecode(Typecode.DATETIME),
        ]):
            return Typecode.STRING

        typecode_list = [
            Typecode.STRING,
            Typecode.FLOAT,
            Typecode.INT,
            Typecode.DATETIME,
            Typecode.BOOL,
            Typecode.INFINITY,
            Typecode.NAN,
        ]

        for typecode in typecode_list:
            if self.__typecode_bitmap & typecode:
                return typecode

        if self.__typecode_bitmap == Typecode.NONE:
            return Typecode.NONE

        return Typecode.STRING

    def __update(self, dataprop):
        self.__str_len = max(self.__str_len, dataprop.str_len)

        if dataprop.typecode in (Typecode.FLOAT, Typecode.INT):
            self.__minmax_integer_digits.update(dataprop.integer_digits)
            self.__minmax_decimal_places.update(dataprop.decimal_places)

        self.__minmax_additional_format_len.update(
            dataprop.additional_format_len)
<|endoftext|>