<commit_before>#!/usr/bin/env python

from __future__ import absolute_import

import os
import sys
import argparse
import importlib

from contextlib import contextmanager

import yaml

from blaze.interactive import data as bz_data
from odo.utils import ignoring

from .server import Server, DEFAULT_PORT

try:
    import __builtin__ as builtins
except ImportError:
    import builtins

__all__ = 'data_spider', 'from_yaml'

def _spider(resource_path, ignore, followlinks, hidden, extra_kwargs):
    resources = {}
    for filename in (os.path.join(resource_path, x)
                     for x in os.listdir(resource_path)):
        basename = os.path.basename(filename)
        if (basename.startswith(os.curdir) and not hidden or
                os.path.islink(filename) and not followlinks):
            continue
        if os.path.isdir(filename):
            new_resources = _spider(filename, ignore=ignore,
                                    followlinks=followlinks,
                                    hidden=hidden,
                                    extra_kwargs=extra_kwargs)
            if new_resources:
                resources[basename] = new_resources
        else:
            with ignoring(*ignore):
                resources[basename] = bz_data(filename, **(extra_kwargs or {}))
    return resources

def data_spider(path,
                ignore=(ValueError, NotImplementedError),
                followlinks=True,
                hidden=False,
                extra_kwargs=None):
    """Traverse a directory and call ``blaze.data`` on its contents.

    Parameters
    ----------
    path : str
        Path to a directory of resources to load
    ignore : tuple of Exception, optional
        Ignore these exceptions when calling ``blaze.data``
    followlinks : bool, optional
        Follow symbolic links
    hidden : bool, optional
        Load hidden files
    extra_kwargs: dict, optional
        extra kwargs to forward on to ``blaze.data``

    Returns
    -------
    dict
        Possibly nested dictionary of containing basenames mapping to resources
    """
    # NOTE: this is named `data_spider` rather than just `spider` to
    # disambiguate this function from the `blaze.server.spider` module.
    return {os.path.basename(path): _spider(path,
                                            ignore=ignore,
                                            followlinks=followlinks,
                                            hidden=hidden,
                                            extra_kwargs=extra_kwargs)}

@contextmanager
def manage_chdir(root):
    """Context manager that changes to ``root`` directory on ``__entry__`` and
    changes back to ``os.getcwd()`` on __exit__.
    """
    root = os.path.abspath(root)
    cwd = os.path.abspath(os.getcwd())
    if root != cwd:
        os.chdir(root)
    yield
    if root != cwd:
        os.chdir(cwd)

def from_yaml(path,
              ignore=(ValueError, NotImplementedError),
              followlinks=True,
              hidden=False,
              relative_to_yaml_dir=False):
    """Construct a dictionary of resources from a YAML specification.

    Parameters
    ----------
    path : file
        File object referring to the YAML specification of resources to load.
    ignore : tuple of Exception, optional
        Ignore these exceptions when calling ``blaze.data``.
    followlinks : bool, optional
        Follow symbolic links.
    hidden : bool, optional
        Load hidden files.
    relative_to_yaml_dir: bool, optional, default False
        Load paths relative to yaml file's directory.  Default is to load
        relative to process' CWD.

    Returns
    -------
    dict
        A dictionary mapping top level keys in a YAML file to resources.

    See Also
    --------
    data_spider : Traverse a directory tree for resources
    """
    resources = {}
    yaml_dir = os.path.dirname(os.path.abspath(path.name))
    for name, info in yaml.load(path.read()).items():
        with manage_chdir(yaml_dir if relative_to_yaml_dir else os.getcwd()):
            try:
                source = info.pop('source')
            except KeyError:
                raise ValueError('source key not found for data source named %r' %
                                 name)
            for mod in info.pop('imports', []):
                importlib.import_module(mod)
            if os.path.isdir(source):
                resources[name] = data_spider(os.path.expanduser(source),
                                              ignore=ignore,
                                              followlinks=followlinks,
                                              hidden=hidden,
                                              extra_kwargs=info)
            else:
                resources[name] = bz_data(source, **info)
    return resources

def _parse_args():
    p = argparse.ArgumentParser(
        formatter_class=argparse.ArgumentDefaultsHelpFormatter)
    p.add_argument('path', type=argparse.FileType('r'), nargs='?',
                   default=sys.stdin,
                   help='A YAML file specifying the resources to load')
    p.add_argument('-p', '--port', type=int, default=DEFAULT_PORT,
                   help='Port number')
    p.add_argument('-H', '--host', type=str, default='127.0.0.1',
                   help='Host name. Use 0.0.0.0 to listen on all public IPs')
    p.add_argument('-l', '--follow-links', action='store_true',
                   help='Follow links when listing files')
    p.add_argument('-e', '--ignored-exception', nargs='+',
                   default=['Exception'],
                   help='Exceptions to ignore when calling ``blaze.data`` on a file')
    p.add_argument('-d', '--hidden', action='store_true',
                   help='Call ``blaze.data`` on hidden files')
    p.add_argument('--yaml-dir', action='store_true',
                   help='Load path-based resources relative to yaml file directory.')
    p.add_argument('-D', '--debug', action='store_true',
                   help='Start the Flask server in debug mode')
    return p.parse_args()

def _main():
    args = _parse_args()
    ignore = tuple(getattr(builtins, e) for e in args.ignored_exception)
    resources = from_yaml(args.path,
                          ignore=ignore,
                          followlinks=args.follow_links,
                          hidden=args.hidden,
                          relative_to_yaml_dir=args.yaml_dir)
    Server(resources).run(host=args.host,
                          port=args.port,
                          debug=args.debug)

if __name__ == '__main__':
    _main()
<commit_msg>Naming and style tweaks after code review.<commit_after>#!/usr/bin/env python

from __future__ import absolute_import

import os
import sys
import argparse
import importlib

from contextlib import contextmanager

import yaml

from blaze.interactive import data as bz_data
from odo.utils import ignoring

from .server import Server, DEFAULT_PORT

try:
    import __builtin__ as builtins
except ImportError:
    import builtins

__all__ = 'data_spider', 'from_yaml'

def _spider(resource_path, ignore, followlinks, hidden, extra_kwargs):
    resources = {}
    for filename in (os.path.join(resource_path, x)
                     for x in os.listdir(resource_path)):
        basename = os.path.basename(filename)
        if (basename.startswith(os.curdir) and not hidden or
                os.path.islink(filename) and not followlinks):
            continue
        if os.path.isdir(filename):
            new_resources = _spider(filename, ignore=ignore,
                                    followlinks=followlinks,
                                    hidden=hidden,
                                    extra_kwargs=extra_kwargs)
            if new_resources:
                resources[basename] = new_resources
        else:
            with ignoring(*ignore):
                resources[basename] = bz_data(filename, **(extra_kwargs or {}))
    return resources

def data_spider(path,
                ignore=(ValueError, NotImplementedError),
                followlinks=True,
                hidden=False,
                extra_kwargs=None):
    """Traverse a directory and call ``blaze.data`` on its contents.

    Parameters
    ----------
    path : str
        Path to a directory of resources to load
    ignore : tuple of Exception, optional
        Ignore these exceptions when calling ``blaze.data``
    followlinks : bool, optional
        Follow symbolic links
    hidden : bool, optional
        Load hidden files
    extra_kwargs: dict, optional
        extra kwargs to forward on to ``blaze.data``

    Returns
    -------
    dict
        Possibly nested dictionary of containing basenames mapping to resources
    """
    # NOTE: this is named `data_spider` rather than just `spider` to
    # disambiguate this function from the `blaze.server.spider` module.
    return {os.path.basename(path): _spider(path,
                                            ignore=ignore,
                                            followlinks=followlinks,
                                            hidden=hidden,
                                            extra_kwargs=extra_kwargs)}

@contextmanager
def pushd(path):
    """Context manager that changes to ``path`` directory on enter and
    changes back to ``os.getcwd()`` on exit.
    """
    path = os.path.abspath(path)
    cwd = os.getcwd()
    if path != cwd:
        os.chdir(path)
    try:
        yield
    finally:
        if path != cwd:
            os.chdir(cwd)

def from_yaml(fh,
              ignore=(ValueError, NotImplementedError),
              followlinks=True,
              hidden=False,
              relative_to_yaml_dir=False):
    """Construct a dictionary of resources from a YAML specification.

    Parameters
    ----------
    fh : file
        File object referring to the YAML specification of resources to load.
    ignore : tuple of Exception, optional
        Ignore these exceptions when calling ``blaze.data``.
    followlinks : bool, optional
        Follow symbolic links.
    hidden : bool, optional
        Load hidden files.
    relative_to_yaml_dir: bool, optional, default False
        Load paths relative to yaml file's directory.  Default is to load
        relative to process' CWD.

    Returns
    -------
    dict
        A dictionary mapping top level keys in a YAML file to resources.

    See Also
    --------
    data_spider : Traverse a directory tree for resources
    """
    resources = {}
    yaml_dir = os.path.dirname(os.path.abspath(fh.name))
    for name, info in yaml.load(fh.read()).items():
        with pushd(yaml_dir if relative_to_yaml_dir else os.getcwd()):
            try:
                source = info.pop('source')
            except KeyError:
                raise ValueError('source key not found for data source named %r' %
                                 name)
            for mod in info.pop('imports', []):
                importlib.import_module(mod)
            if os.path.isdir(source):
                resources[name] = data_spider(os.path.expanduser(source),
                                              ignore=ignore,
                                              followlinks=followlinks,
                                              hidden=hidden,
                                              extra_kwargs=info)
            else:
                resources[name] = bz_data(source, **info)
    return resources

def _parse_args():
    p = argparse.ArgumentParser(
        formatter_class=argparse.ArgumentDefaultsHelpFormatter)
    p.add_argument('path', type=argparse.FileType('r'), nargs='?',
                   default=sys.stdin,
                   help='A YAML file specifying the resources to load')
    p.add_argument('-p', '--port', type=int, default=DEFAULT_PORT,
                   help='Port number')
    p.add_argument('-H', '--host', type=str, default='127.0.0.1',
                   help='Host name. Use 0.0.0.0 to listen on all public IPs')
    p.add_argument('-l', '--follow-links', action='store_true',
                   help='Follow links when listing files')
    p.add_argument('-e', '--ignored-exception', nargs='+',
                   default=['Exception'],
                   help='Exceptions to ignore when calling ``blaze.data`` on a file')
    p.add_argument('-d', '--hidden', action='store_true',
                   help='Call ``blaze.data`` on hidden files')
    p.add_argument('--yaml-dir', action='store_true',
                   help='Load path-based resources relative to yaml file directory.')
    p.add_argument('-D', '--debug', action='store_true',
                   help='Start the Flask server in debug mode')
    return p.parse_args()

def _main():
    args = _parse_args()
    ignore = tuple(getattr(builtins, e) for e in args.ignored_exception)
    resources = from_yaml(args.path,
                          ignore=ignore,
                          followlinks=args.follow_links,
                          hidden=args.hidden,
                          relative_to_yaml_dir=args.yaml_dir)
    Server(resources).run(host=args.host,
                          port=args.port,
                          debug=args.debug)

if __name__ == '__main__':
    _main()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

"""
Minimal python JSON-RPC 2.0 implementation in a single file.
"""

__author__     = "Marcel Rieger"
__copyright__  = "Copyright 2016, Marcel Rieger"
__credits__    = ["Marcel Rieger"]
__contact__    = "https://github.com/riga/jsonrpyc"
__license__    = "MIT"
__status__     = "Development"
__version__    = "0.0.1"

__all__ = ["RPC"]

import sys
import json
import io
import time
import threading

class Spec(object):

    @classmethod
    def check_id(cls, id, allow_empty=False):
        if allow_empty and id is None:
            return
        if not isinstance(id, int):
            raise TypeError("id must be an integer, got %s (%s)" % (id, type(id)))

    @classmethod
    def check_method(cls, method):
        if not isinstance(method, str):
            raise TypeError("method must be a string, got %s (%s)" % (method, type(method)))

    @classmethod
    def check_code(cls, code):
        if not isinstance(code, int):
            raise TypeError("code must be an integer, got %s (%s)" % (id, type(id)))
        elif get_error(code) is None:
            raise KeyError("unknown code, got %s (%s)" % (code, type(code)))

    @classmethod
    def request(cls, method, id=None, params=None):
        try:
            cls.check_method(method)
            cls.check_id(id, allow_empty=True)
        except Exception as e:
            raise RPCInvalidRequest(str(e))

        req = "{\"jsonrpc\":\"2.0\",\"method\":\"%s\"" % method

        if id is not None:
            req += ",\"id\":%d" % id

        if params is not None:
            try:
                req += ",\"params\":%s" % json.dumps(params)
            except Exception as e:
                raise RPCParseError(str(e))

        req += "}"

        return req

    @classmethod
    def response(cls, id, result):
        try:
            cls.check_id(id)
        except Exception as e:
            raise RPCInvalidRequest(str(e))

        try:
            res = "{\"jsonrpc\":\"2.0\",\"id\":%d,\"result\":%s}" % (id, json.dumps(result))
        except Exception as e:
            raise RPCParseError(str(e))

        return res

    @classmethod
    def error(cls, id, code, data=None):
        try:
            cls.check_id(id)
            cls.check_code(code)
        except Exception as e:
            raise RPCInvalidRequest(str(e))

        message = get_error(code).title

        err = "{\"code\":%d,\"message\":\"%s\"" % (code, message)

        if data is not None:
            try:
                err += ",\"data\":%s}" % json.dumps(data)
            except Exception as e:
               raise RPCParseError(str(e))
        else:
            err += "}"

        err = "{\"jsonrpc\":\"2.0\",\"id\":%d,\"error\":%s}" % (id, err)

        return err

class RPC(object):

    EMPTY_RESULT = object()

    def __init__(self, target=None, stdin=None, stdout=None, **kwargs):
        super(RPC, self).__init__()

        self.target = target

        stdin = sys.stdin if stdin is None else stdin
        stdout = sys.stdout if stdout is None else stdout

        self.stdin = io.open(stdin.fileno(), "rb")
        self.stdout = io.open(stdout.fileno(), "wb")

        self._i = -1
        self._callbacks = {}
        self._results = {}

        kwargs.setdefault("daemon", target is None)
        self._watchdog = Watchdog(self, **kwargs)

    def __call__(self, *args, **kwargs):
        return self.call(*args, **kwargs)

    def call(self, method, args=(), kwargs=None, callback=None, block=0):
        if kwargs is None:
            kwargs = {}

        if callback is not None or block > 0:
            self._i += 1
            id = self._i
        else:
            id = None

        if callback is not None:
            self._callbacks[id] = callback

        if block > 0:
            self._results[id] = self.EMPTY_RESULT

        params = {"args": args, "kwargs": kwargs}
        req = Spec.request(method, id=id, params=params)
        self._write(req)

        if block > 0:
            while True:
                if self._results[id] != self.EMPTY_RESULT:
                    result = self._results[id]
                    del self._results[id]
                    if isinstance(result, Exception):
                        raise result
                    else:
                        return result
                time.sleep(block)

    def _handle(self, line):
        obj = json.loads(line)

        # dispatch to the correct handler
        if "method" in obj:
            # request
            self._handle_request(obj)
        elif "error" not in obj:
            # response
            self._handle_response(obj)
        else:
            # error
            self._handle_error(obj)

    def _handle_request(self, req):
        try:
            method = self._route(req["method"])
            result = method(*req["params"]["args"], **req["params"]["kwargs"])
            if "id" in req:
                res = Spec.response(req["id"], result)
                self._write(res)
        except Exception as e:
            if "id" in req:
                if isinstance(e, RPCError):
                    err = Spec.error(req["id"], e.code, e.data)
                else:
                    err = Spec.error(req["id"], -32603, str(e))
                self._write(err)

    def _handle_response(self, res):
        if res["id"] in self._results:
            self._results[res["id"]] = res["result"]
        if res["id"] in self._callbacks:
            callback = self._callbacks[res["id"]]
            del self._callbacks[res["id"]]
            callback(None, res["result"])

    def _handle_error(self, res):
        err = res["error"]
        error = get_error(err["code"])(err.get("data", err["message"]))

        if res["id"] in self._results:
            self._results[res["id"]] = error
        if res["id"] in self._callbacks:
            callback = self._callbacks[res["id"]]
            del self._callbacks[res["id"]]
            callback(error, None)

    def _route(self, method):
        obj = self.target
        for part in method.split("."):
            if not hasattr(obj, part):
                break
            obj = getattr(obj, part)
        else:
            return obj
        raise RPCMethodNotFound(data=method)

    def _write(self, s):
        self.stdout.write(bytearray(s + "\n", "utf-8"))
        self.stdout.flush()

class Watchdog(threading.Thread):

    def __init__(self, rpc, name=None, interval=0.1, daemon=False, start=True):
        super(Watchdog, self).__init__()

        self.rpc = rpc
        self.name = name
        self.interval = interval
        self.daemon = daemon

        self._stop = threading.Event()

        if start:
            self.start()

    def run(self):
        stream = self.rpc.stdin

        if stream.isatty():
            last_pos = 0
            while not self._stop.is_set():
                cur_pos = stream.tell()
                if cur_pos != last_pos:
                    stream.seek(last_pos)
                    lines = stream.readlines()
                    last_pos = stream.tell()
                    stream.seek(cur_pos)
                    for line in lines:
                        line = line.decode("utf-8").strip()
                        if line:
                            self.rpc._handle(line)
                else:
                    self._stop.wait(self.interval)
        else:
            while not self._stop.is_set():
                line = stream.readline().decode("utf-8").rstrip()
                if line:
                    self.rpc._handle(line)
                else:
                    self._stop.wait(self.interval)

    def stop(self):
        self._stop.set()

class RPCError(Exception):

    def __init__(self, data=None):
        message = "%s (%s)" % (self.title, self.code)
        if data is not None:
            message += ", data: " + str(data)
        self.message = message

        super(RPCError, self).__init__(message)

        self.data = data

    def __str__(self):
        return self.message

error_map_distinct = {}
error_map_range = {}

def is_range(code):
    return isinstance(code, tuple) and len(code) == 2 and all(isinstance(i, int) for i in code) \
           and code[0] < code[1]

def register_error(cls):
    # it would be much cleaner to add a meta class to RPCError as a registry for codes
    # but in cpython exceptions aren't types, so simply provide a registry mechanism here
    code = cls.code

    if isinstance(code, int):
        error_map = error_map_distinct
    elif is_range(code):
        error_map = error_map_range
    else:
        raise ValueError("invalid RPC error code " + str(code))

    if code in error_map:
        raise AttributeError("duplicate RPC error code " + str(code))

    error_map[code] = cls

    return cls

def get_error(code):
    if code in error_map_distinct:
        return error_map_distinct[code]

    for (lower, upper), cls in error_map_range.items():
        if lower <= code <= upper:
            return cls

    return None

@register_error
class RPCParseError(RPCError):

    code = -32700
    title = "Parse error"

@register_error
class RPCInvalidRequest(RPCError):

    code = -32600
    title = "Invalid Request"

@register_error
class RPCMethodNotFound(RPCError):

    code = -32601
    title = "Method not found"

@register_error
class RPCInvalidParams(RPCError):

    code = -32602
    title = "Invalid params"

@register_error
class RPCInternalError(RPCError):

    code = -32603
    title = "Internal error"

@register_error
class RPCServerError(RPCError):

    code = (-32099, -32000)
    title = "Server error"
<commit_msg>Add doc strings, closes #2.<commit_after># -*- coding: utf-8 -*-

"""
Minimal python RPC implementation in a single file based on the
`JSON-RPC 2.0 specs <http://www.jsonrpc.org/specification>`_.
"""

__author__     = "Marcel Rieger"
__copyright__  = "Copyright 2016, Marcel Rieger"
__credits__    = ["Marcel Rieger"]
__contact__    = "https://github.com/riga/jsonrpyc"
__license__    = "MIT"
__status__     = "Development"
__version__    = "0.0.1"

__all__ = ["RPC"]

import sys
import json
import io
import time
import threading

class Spec(object):
    """
    This class wraps methods that create JSON-RPC 2.0 compatible string representations of
    requests, responses, and errors. All methods are class members, so you might never want to
    create an instance of this class, but rahter use the methods directly:

    .. code-block:: python

       Spec.request("my_method", 18)
       # => '{"jsonrpc": "2.0", "method": "my_method", "id": 18}'

       Spec.response(18, "some_result")
       # => '{"jsonrpc": "2.0", "id": 18, "result": "some_result"}'

       Spec.error(18, -32603)
       # => '{"jsonrpc": "2.0", "id": 18, "error": {"code": -32603, "message": "Internal error"}}'
    """

    @classmethod
    def check_id(cls, id, allow_empty=False):
        """
        Value check for *id* entries. When *allow_empty* is *True*, *id* is allowed to be *None*.
        Raises a *TypeError* when *id* is not an interger.
        """
        if allow_empty and id is None:
            return
        if not isinstance(id, int):
            raise TypeError("id must be an integer, got %s (%s)" % (id, type(id)))

    @classmethod
    def check_method(cls, method):
        """
        Value check for *method* entries. Raises a *TypeError* when *method* is not a string.
        """
        if not isinstance(method, str):
            raise TypeError("method must be a string, got %s (%s)" % (method, type(method)))

    @classmethod
    def check_code(cls, code):
        """
        Value check for *code* entries. Raises a *TypeError* when *code* is not an interger, or a
        *KeyError* when there is no :py:class:`RPCError` derivative registered for that *code*.
        """
        if not isinstance(code, int):
            raise TypeError("code must be an integer, got %s (%s)" % (id, type(id)))
        elif get_error(code) is None:
            raise KeyError("unknown code, got %s (%s)" % (code, type(code)))

    @classmethod
    def request(cls, method, id=None, params=None):
        """
        Creates the string representation of a request that calls *method* with optional *params*.
        When *id* is *None*, the request is considered a message.
        """
        try:
            cls.check_method(method)
            cls.check_id(id, allow_empty=True)
        except Exception as e:
            raise RPCInvalidRequest(str(e))

        req = "{\"jsonrpc\":\"2.0\",\"method\":\"%s\"" % method

        if id is not None:
            req += ",\"id\":%d" % id

        if params is not None:
            try:
                req += ",\"params\":%s" % json.dumps(params)
            except Exception as e:
                raise RPCParseError(str(e))

        req += "}"

        return req

    @classmethod
    def response(cls, id, result):
        """
        Creates the string representation of a respons that was triggered by a request with *id*.
        *result* is required.
        """
        try:
            cls.check_id(id)
        except Exception as e:
            raise RPCInvalidRequest(str(e))

        try:
            res = "{\"jsonrpc\":\"2.0\",\"id\":%d,\"result\":%s}" % (id, json.dumps(result))
        except Exception as e:
            raise RPCParseError(str(e))

        return res

    @classmethod
    def error(cls, id, code, data=None):
        """
        Creates the string representation of an error that occured while processing a request with
        *id*. *code* must lead to a registered :py:class:`RPCError`. *data* might contain
        additional, detailed error information.
        """
        try:
            cls.check_id(id)
            cls.check_code(code)
        except Exception as e:
            raise RPCInvalidRequest(str(e))

        message = get_error(code).title

        err = "{\"code\":%d,\"message\":\"%s\"" % (code, message)

        if data is not None:
            try:
                err += ",\"data\":%s}" % json.dumps(data)
            except Exception as e:
               raise RPCParseError(str(e))
        else:
            err += "}"

        err = "{\"jsonrpc\":\"2.0\",\"id\":%d,\"error\":%s}" % (id, err)

        return err

class RPC(object):
    """
    The main class of *jsonrpyc*. Instances of this class basically wrap an input stream *stdin* and
    an output stream *stdout* in order to communicate with other *objects*, *processes* or
    *services* that implement the JSON-RPC 2.0 specs. RPC instances may wrap a *target* object.
    Incomming requests will be routed to methods of this object whose result might be sent back as a
    response. Example implementation:

    *server.py*

    .. code-block:: python

       import jsonrpyc

       class MyTarget(object):

           def greet(self, name):
               return "Hi, %s!" % name

       jsonrpc.RPC(MyTarget())

    *client.py*

    .. code-block:: python

        import jsonrpyc
        from subprocess import Popen, PIPE

        p = Popen(["python", "server.py"], stdin=PIPE, stdout=PIPE)
        rpc = jsonrpyc.RPC(stdout=p.stdin, stdin=p.stdout)

        # non-blocking remote procedure call with callback, js-like signature
        def cb(err, res=None):
            if err:
                throw err
            print("callback got: " + res)
        rpc("greet", args=("John",))
        # cb is called asynchronously which prints
        # => "callback got: Hi, John!"

        # blocking remote procedure call with 0.1s polling
        print(rpc("greet", args=("John",), block=0.1))
        # => "Hi, John!"

    .. py:attribute:: target

       The wrapped target object. Might be *None* when no object is wrapped, e.g. for the *client*
       RPC instance.

    .. py:attribute:: stdin

       The input stream, re-opened with ``"rb"``.

    .. py:attribute:: stdout

       The output stream, re-opened with ``"wb"``.

    .. py:attribute:: watch

       The :py:class:`Watchdog` instance that watches *stdin*.
    """

    EMPTY_RESULT = object()

    def __init__(self, target=None, stdin=None, stdout=None, **kwargs):
        super(RPC, self).__init__()

        self.target = target

        stdin = sys.stdin if stdin is None else stdin
        stdout = sys.stdout if stdout is None else stdout

        self.stdin = io.open(stdin.fileno(), "rb")
        self.stdout = io.open(stdout.fileno(), "wb")

        self._i = -1
        self._callbacks = {}
        self._results = {}

        kwargs.setdefault("daemon", target is None)
        self.watchdog = Watchdog(self, **kwargs)

    def __call__(self, *args, **kwargs):
        return self.call(*args, **kwargs)

    def call(self, method, args=(), kwargs=None, callback=None, block=0):
        """
        Performs an actual remote procedure call by writing a request representation (a string) to
        the output stream. The remote RPC instance uses *method* to route to the actual method to
        call with *args* and *kwargs*. When *callback* is set, it will be called with the result of
        the remote call. When *block* is larger than *0*, the calling thread is blocked until the
        result is received. In this case, *block* will be the poll interval. This mechanism emulates
        synchronuous return value behavior. When both, *callback* is *None* and *block* is *0* or
        less, the request is considered a message and the remote RPC instance will not send a
        response.
        """
        if kwargs is None:
            kwargs = {}

        if callback is not None or block > 0:
            self._i += 1
            id = self._i
        else:
            id = None

        if callback is not None:
            self._callbacks[id] = callback

        if block > 0:
            self._results[id] = self.EMPTY_RESULT

        params = {"args": args, "kwargs": kwargs}
        req = Spec.request(method, id=id, params=params)
        self._write(req)

        if block > 0:
            while True:
                if self._results[id] != self.EMPTY_RESULT:
                    result = self._results[id]
                    del self._results[id]
                    if isinstance(result, Exception):
                        raise result
                    else:
                        return result
                time.sleep(block)

    def _handle(self, line):
        """
        Handles an incomming *line* and dispatches the parsed object to the request, response, or
        error handler.
        """
        obj = json.loads(line)

        # dispatch to the correct handler
        if "method" in obj:
            # request
            self._handle_request(obj)
        elif "error" not in obj:
            # response
            self._handle_response(obj)
        else:
            # error
            self._handle_error(obj)

    def _handle_request(self, req):
        """
        Handles an incomming request *req*. When it containes an id, a response or error is sent
        back.
        """
        try:
            method = self._route(req["method"])
            result = method(*req["params"]["args"], **req["params"]["kwargs"])
            if "id" in req:
                res = Spec.response(req["id"], result)
                self._write(res)
        except Exception as e:
            if "id" in req:
                if isinstance(e, RPCError):
                    err = Spec.error(req["id"], e.code, e.data)
                else:
                    err = Spec.error(req["id"], -32603, str(e))
                self._write(err)

    def _handle_response(self, res):
        """
        Handles an incomming successful response *res*. Blocking calls are resolved and registered
        callbacks are invoked with the first error argument being set to *None*.
        """
        if res["id"] in self._results:
            self._results[res["id"]] = res["result"]
        if res["id"] in self._callbacks:
            callback = self._callbacks[res["id"]]
            del self._callbacks[res["id"]]
            callback(None, res["result"])

    def _handle_error(self, res):
        """
        Handles an incomming failed response *res*. Blocking calls throw an exception and
        registered callbacks are invoked with an exception and the second result argument set to
        *None*.
        """
        err = res["error"]
        error = get_error(err["code"])(err.get("data", err["message"]))

        if res["id"] in self._results:
            self._results[res["id"]] = error
        if res["id"] in self._callbacks:
            callback = self._callbacks[res["id"]]
            del self._callbacks[res["id"]]
            callback(error, None)

    def _route(self, method):
        """
        Returnes the actual method of the wrapped target object to be called when *method* is
        requested. Example:

        .. code-block:: python
           MyClassB(object):
               def foo(self):
                   return 123

           MyClassA(object):
               def __init__(self):
                   self.b = MyClassB()
               def bar(self):
                   return "test"

           rpc = RPC(MyClassA())

           rpc._route("bar")
           # => <bound method MyClassA.bar ...>

           rpc._route("b.foo")
           # => <bound method MyClassb.foo ...>
        """
        obj = self.target
        for part in method.split("."):
            if not hasattr(obj, part):
                break
            obj = getattr(obj, part)
        else:
            return obj
        raise RPCMethodNotFound(data=method)

    def _write(self, s):
        """
        Writes a string *s* to the output stream.
        """
        self.stdout.write(bytearray(s + "\n", "utf-8"))
        self.stdout.flush()

class Watchdog(threading.Thread):
    """
    This class represents a thread that watches the input stream for incomming content.

    .. py:attribute:: rpc

       The :py:class:`RPC` instance.

    .. py:attribute:: name

       The thread's name.

    .. py:attribute:: interval

       The polling interval of the run loop.

    .. py:attribute:: daemon

       The thread's daemon flag.
    """

    def __init__(self, rpc, name="watchdog", interval=0.1, daemon=False, start=True):
        super(Watchdog, self).__init__()

        self.rpc = rpc
        self.name = name
        self.interval = interval
        self.daemon = daemon

        self._stop = threading.Event()

        if start:
            self.start()

    def start(self):
        """
        Starts with thread's activity.
        """
        super(Watchdog, self).start()

    def stop(self):
        """
        Stops with thread's activity.
        """
        self._stop.set()

    def run(self):
        stream = self.rpc.stdin

        if stream.isatty():
            last_pos = 0
            while not self._stop.is_set():
                cur_pos = stream.tell()
                if cur_pos != last_pos:
                    stream.seek(last_pos)
                    lines = stream.readlines()
                    last_pos = stream.tell()
                    stream.seek(cur_pos)
                    for line in lines:
                        line = line.decode("utf-8").strip()
                        if line:
                            self.rpc._handle(line)
                else:
                    self._stop.wait(self.interval)
        else:
            while not self._stop.is_set():
                line = stream.readline().decode("utf-8").rstrip()
                if line:
                    self.rpc._handle(line)
                else:
                    self._stop.wait(self.interval)

class RPCError(Exception):

    """
    Base class for RPC errors.

    .. py:attribute:: message

       The message of this error, i.e., ``"<title> (<code>)[, data: <data>]"``.

    .. py:attribute:: data

       Additional data of this error. Setting the data attribute will also change the message
       attribute.
    """

    def __init__(self, data=None):
        message = "%s (%s)" % (self.title, self.code)
        if data is not None:
            message += ", data: " + str(data)
        self.message = message

        super(RPCError, self).__init__(message)

        self.data = data

    def __str__(self):
        return self.message

error_map_distinct = {}
error_map_range = {}

def is_range(code):
    return isinstance(code, tuple) and len(code) == 2 and all(isinstance(i, int) for i in code) \
           and code[0] < code[1]

def register_error(cls):
    """
    Decorator that registers a new RPC error derived from :py:class:`RPCError`. The purpose of
    error registration is to have a mapping of error codes/code ranges to error classes for faster
    lookups during error creation.

    .. code-block:: python

       @register_error
       class MyCustomRPCError(RPCError):
           code = ...
           title = "My custom error"
    """
    # it would be much cleaner to add a meta class to RPCError as a registry for codes
    # but in cpython exceptions aren't types, so simply provide a registry mechanism here
    if not issubclass(cls, RPCError):
        raise TypeError("'%s' is not a subclass of RPCError" % cls)

    code = cls.code

    if isinstance(code, int):
        error_map = error_map_distinct
    elif is_range(code):
        error_map = error_map_range
    else:
        raise ValueError("invalid RPC error code " + str(code))

    if code in error_map:
        raise AttributeError("duplicate RPC error code " + str(code))

    error_map[code] = cls

    return cls

def get_error(code):
    """
    Returns the RPC error class that was previously registered to *code*. *None* is returned when no
    class could be found.
    """
    if code in error_map_distinct:
        return error_map_distinct[code]

    for (lower, upper), cls in error_map_range.items():
        if lower <= code <= upper:
            return cls

    return None

@register_error
class RPCParseError(RPCError):

    code = -32700
    title = "Parse error"

@register_error
class RPCInvalidRequest(RPCError):

    code = -32600
    title = "Invalid Request"

@register_error
class RPCMethodNotFound(RPCError):

    code = -32601
    title = "Method not found"

@register_error
class RPCInvalidParams(RPCError):

    code = -32602
    title = "Invalid params"

@register_error
class RPCInternalError(RPCError):

    code = -32603
    title = "Internal error"

@register_error
class RPCServerError(RPCError):

    code = (-32099, -32000)
    title = "Server error"
<|endoftext|>


<commit_before># Django settings for testproject project.

DEBUG = True
TEMPLATE_DEBUG = DEBUG
DEBUG_PROPAGATE_EXCEPTIONS = True

ALLOWED_HOSTS = ['*']

ADMINS = tuple(
    # ('Your Name', 'your_email@domain.com'),
)

MANAGERS = ADMINS

DATABASES = {
    'default': {
        'ENGINE': 'django.db.backends.sqlite3',
        'NAME': 'backupdb_database',
        'TEST_NAME': 'test_backupdb_database',
    },
    'mysql': {
        'ENGINE': 'django.db.backends.mysql',
        'NAME': 'backupdb',
        'USER': 'root',
    },
    'postgresql': {
        'ENGINE': 'django.db.backends.postgresql_psycopg2',
        'NAME': 'backupdb',
        'USER': 'backupdb_user',
    },
}

CACHES = {
    'default': {
        'BACKEND': 'django.core.cache.backends.locmem.LocMemCache',
    }
}

# Local time zone for this installation. Choices can be found here:
# http://en.wikipedia.org/wiki/List_of_tz_zones_by_name
# although not all choices may be available on all operating systems.
# On Unix systems, a value of None will cause Django to use the same
# timezone as the operating system.
# If running in a Windows environment this must be set to the same as your
# system time zone.
TIME_ZONE = 'America/Denver'

# Language code for this installation. All choices can be found here:
# http://www.i18nguy.com/unicode/language-identifiers.html
LANGUAGE_CODE = 'en-us'

SITE_ID = 1

# If you set this to False, Django will make some optimizations so as not
# to load the internationalization machinery.
USE_I18N = True

# If you set this to False, Django will not format dates, numbers and
# calendars according to the current locale
USE_L10N = True

# Absolute filesystem path to the directory that will hold user-uploaded files.
# Example: "/home/media/media.lawrence.com/"
MEDIA_ROOT = ''

# URL that handles the media served from MEDIA_ROOT. Make sure to use a
# trailing slash if there is a path component (optional in other cases).
# Examples: "http://media.lawrence.com", "http://example.com/media/"
MEDIA_URL = ''

# Make this unique, and don't share it with anybody.
SECRET_KEY = 'kygee734_14^(r-p&1j1_&)n1&9-efw0_@&*3nu7)zh%iu%fm1'

# List of callables that know how to import templates from various sources.
TEMPLATE_LOADERS = (
    'django.template.loaders.filesystem.Loader',
    'django.template.loaders.app_directories.Loader',
    # 'django.template.loaders.eggs.Loader',
)

MIDDLEWARE_CLASSES = (
    'django.middleware.common.CommonMiddleware',
    'django.contrib.sessions.middleware.SessionMiddleware',
    'django.middleware.csrf.CsrfViewMiddleware',
    'django.contrib.auth.middleware.AuthenticationMiddleware',
    'django.contrib.messages.middleware.MessageMiddleware',
)

ROOT_URLCONF = 'urls'

TEMPLATE_DIRS = (
    # Put strings here, like "/home/html/django_templates" or "C:/www/django/templates".
    # Always use forward slashes, even on Windows.
    # Don't forget to use absolute paths, not relative paths.
)

INSTALLED_APPS = (
    'django.contrib.auth',
    'django.contrib.contenttypes',
    'django.contrib.sessions',
    'django.contrib.sites',
    'django.contrib.messages',
    # Uncomment the next line to enable the admin:
    # 'django.contrib.admin',
    # Uncomment the next line to enable admin documentation:
    # 'django.contrib.admindocs',
    'backupdb',
    'backupdb.tests',
)

STATIC_URL = '/static/'
<commit_msg>Update tests settings<commit_after>DEBUG = True
TEMPLATE_DEBUG = DEBUG
DEBUG_PROPAGATE_EXCEPTIONS = True

ALLOWED_HOSTS = ['*']

ADMINS = tuple()

MANAGERS = ADMINS

DATABASES = {
    'default': {
        'ENGINE': 'django.db.backends.sqlite3',
        'NAME': 'backupdb_database',
    },
    'mysql': {
        'ENGINE': 'django.db.backends.mysql',
        'NAME': 'backupdb',
        'USER': 'root',
    },
    'postgresql': {
        'ENGINE': 'django.db.backends.postgresql_psycopg2',
        'NAME': 'backupdb',
        'USER': 'backupdb_user',
    },
}

CACHES = {
    'default': {
        'BACKEND': 'django.core.cache.backends.locmem.LocMemCache',
    }
}

TIME_ZONE = 'America/Denver'
LANGUAGE_CODE = 'en-us'

SITE_ID = 1

USE_I18N = True
USE_L10N = True

MEDIA_ROOT = ''

MEDIA_URL = ''

SECRET_KEY = 'kygee734_14^(r-p&1j1_&)n1&9-efw0_@&*3nu7)zh%iu%fm1'

TEMPLATE_LOADERS = (
    'django.template.loaders.filesystem.Loader',
    'django.template.loaders.app_directories.Loader',
)

MIDDLEWARE_CLASSES = (
    'django.middleware.common.CommonMiddleware',
    'django.contrib.sessions.middleware.SessionMiddleware',
    'django.middleware.csrf.CsrfViewMiddleware',
    'django.contrib.auth.middleware.AuthenticationMiddleware',
    'django.contrib.messages.middleware.MessageMiddleware',
)

ROOT_URLCONF = 'urls'

INSTALLED_APPS = (
    'django.contrib.auth',
    'django.contrib.contenttypes',
    'django.contrib.sessions',
    'django.contrib.sites',
    'django.contrib.messages',

    'backupdb',
    'backupdb.tests',
)

STATIC_URL = '/static/'
<|endoftext|>


<commit_before># coding: utf-8
# Copyright 2013 The Font Bakery Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# See AUTHORS.txt for the list of Authors and LICENSE.txt for the License.
from __future__ import print_function
from collections import defaultdict, Counter, namedtuple, OrderedDict
import os
import os.path as op
import yaml

from fontaine.cmap import Library
from fontaine.font import FontFactory

from bakery_cli.scripts.vmet import get_metric_view
from bakery_cli.utils import UpstreamDirectory

from bakery_cli.report import utils as report_utils

from bakery_lint.metadata import Metadata

TAB = 'Index'
TEMPLATE_DIR = op.join(op.dirname(__file__), 'templates')

t = lambda templatefile: op.join(TEMPLATE_DIR, templatefile)

def sort(data):
    a = []
    for grouped_dict in data:
        if 'required' in grouped_dict['tags']:
            a.append(grouped_dict)

    for grouped_dict in data:
        if 'note' in grouped_dict['tags'] and 'required' not in grouped_dict['tags']:
            a.append(grouped_dict)

    for grouped_dict in data:
        if 'note' not in grouped_dict['tags'] and 'required' not in grouped_dict['tags']:
            a.append(grouped_dict)

    return a

def filter_with_tag(fonttestdata, tag):
    tests = fonttestdata['failure'] + fonttestdata['error']
    return [test for test in tests if tag in test['tags']]

def filter_by_results_with_tag(fonttestdata, tag, *results):
    tests = {}
    for res in results:
        tests[res] = [test for test in fonttestdata.get(res) if tag in test.get('tags', [])]
    return tests

def get_fonts_table_sizes(fonts):
    """ Returns tuple with available tables from all fonts and their length """
    from fontTools.ttLib import sfnt
    _fonts = {}
    tables = []
    for font in fonts:
        _fonts[op.basename(font)] = {}
        with open(font) as fp_font:
            sf = sfnt.SFNTReader(fp_font)
            for t in sf.tables:
                if t not in tables:
                    tables.append(t)
                _fonts[op.basename(font)][t] = sf.tables[t].length
    return tables, _fonts

def get_fonts_table_sizes_grouped(fonts_list):
    _, fonts = get_fonts_table_sizes(fonts_list)
    fonts_dict = defaultdict(dict, fonts)

    # Fonts may have different tables!!!

    # across all fonts calculate sum of each table
    table_sizes_sums = sum(
        (Counter(v) for k, v in fonts_dict.iteritems()), Counter()
    )

    # count amount of each table across all fonts
    tables_counts = sum(
        (Counter(v.keys()) for k, v in fonts_dict.iteritems()), Counter()
    )

    # count average for each table, take value from 'table_sizes_sums'
    # and divide by corresponding value from  'tables_counts',
    # eg table_sizes_sums['glyf'] / tables_counts['glyf']
    tables_mean_dict = {
        k: table_sizes_sums[k]/tables_counts[k] for k in table_sizes_sums
    }

    # calculate deviation (delta) from an average
    # for each font and each table in font find delta
    tables_delta_dict = {}
    for font, tables in fonts_dict.iteritems():
        tables_delta_dict[font] = {
            k: tables_mean_dict[k]-v for k, v in tables.iteritems()
        }

    # gather all existent tables from all fonts
    all_possible_tables = set()
    for font, tables in tables_delta_dict.items():
        for table in tables:
            if table not in all_possible_tables:
                all_possible_tables.add(table)

    # if some font does not have a table that others have,
    # just set the deviation to 0
    for font, tables in tables_delta_dict.items():
        for item in all_possible_tables:
            tables.setdefault(item, 0)
        tables_delta_dict[font] = tables

    # make the deviation dict ready for google chart as array
    tables_delta_dict_for_google_array = {}
    for font, props in tables_delta_dict.iteritems():
        tables_delta_dict_for_google_array.setdefault('fonts', []).append(font)
        for k, v in props.iteritems():
            tables_delta_dict_for_google_array.setdefault(k, []).append(v)

    # prepare all tables dict as array for google chart
    tables_dict_for_google_array = {}
    for font, props in fonts.iteritems():
        tables_dict_for_google_array.setdefault('fonts', []).append(font)
        for k, v in props.iteritems():
            tables_dict_for_google_array.setdefault(k, []).append(v)

    grouped_dict = {
        'fonts': tables_dict_for_google_array.pop('fonts'),
        'tables': [
            [k, tables_mean_dict[k]] + v for k, v in tables_dict_for_google_array.items()
        ]
    }

    delta_dict = {
        'fonts': tables_delta_dict_for_google_array.pop('fonts'),
        'tables': [
            [k, ] + v for k, v in tables_delta_dict_for_google_array.items()
        ]
    }

    # make all arrays to have same len
    max_len = len(max(grouped_dict['tables'], key=len))
    new_items = []
    for item in grouped_dict["tables"]:
        new_item = item[:]
        while len(new_item) < max_len:
            new_item.append(0)
        new_items.append(new_item)
    grouped_dict["tables"] = new_items

    ftable = namedtuple('FontTable', ['mean', 'grouped', 'delta'])
    return ftable(tables_mean_dict, grouped_dict, delta_dict)

def get_orthography(fontaineFonts):
    fonts_dict = defaultdict(list)
    library = Library(collections=['subsets'])
    fonts_names = []
    for font, fontaine in fontaineFonts:
        fonts_names.append(font)
        for info in fontaine.get_orthographies(_library=library):
            font_info = dict(name=font, support=info.support_level,
                             coverage=info.coverage,
                             missing_chars=info.missing)
            fonts_dict[info.charmap.common_name].append(font_info)
    averages = {}
    for subset, fonts in fonts_dict.items():
        averages[subset] = sum([font['coverage'] for font in fonts]) / len(fonts)
    return sorted(fonts_names), averages, OrderedDict(sorted(fonts_dict.items()))

def to_google_data_list(tdict, haxis=0):
    return sorted([[x, tdict[x] - haxis] for x in tdict])

def font_table_to_google_data_list(tdict):
    return sorted([list(item) for item in tdict.items()])

def average_table_size(tdict):
    return sum(tdict.values()) / len(tdict)

def _obj_to_dict(instance, exclude_attrs=()):
    # Very simplified, but enough for reports
    return {
        k: getattr(instance, k) for k in dir(instance) \
        if not any([k.startswith('__'), str(k) in exclude_attrs])
    }

def font_factory_instance_to_dict(instance):
    return _obj_to_dict(instance, exclude_attrs=(
        'get_othography_info', 'get_orthographies', 'orthographies',
        'refresh_sfnt_properties', '_fontFace', 'getGlyphNames'
    ))

def generate(config):
    if config.get('failed'):
        return

    directory = UpstreamDirectory(config['path'])
    if op.exists(op.join(config['path'], 'METADATA.new.json')):
        metadata_file = open(op.join(config['path'], 'METADATA.new.json')).read()
    else:
        metadata_file = open(op.join(config['path'], 'METADATA.json')).read()
    family_metadata = Metadata.get_family_metadata(metadata_file)
    faces = []
    for f in family_metadata.fonts:
        faces.append({'name': f.full_name,
                      'basename': f.post_script_name,
                      'path': f.filename,
                      'meta': f})

    metadata = yaml.load(open(op.join(config['path'], 'METADATA.yaml')))
    upstreamdata = {}
    upstreamdatafile = op.join(config['path'], 'upstream.yaml')
    if op.exists(upstreamdatafile):
        upstreamdata = yaml.load(open(upstreamdatafile))

    data = {}
    for fp in directory.BIN:
        path = op.join(config['path'], '{}.yaml'.format(fp[:-4]))
        if op.exists(path):
            data[fp] = yaml.load(open(path))
    data.update(metadata)
    data.update(upstreamdata)

    fontpaths = [op.join(config['path'], path)
                 for path in directory.BIN]
    ttftablesizes = get_fonts_table_sizes(fontpaths)

    ftables_data = get_fonts_table_sizes_grouped(fontpaths)

    buildstate = yaml.load(open(op.join(config['path'], 'build.state.yaml')))
    autohint_sizes = buildstate.get('autohinting_sizes', [])
    vmet = get_metric_view(fontpaths)

    fonts = [(path, FontFactory.openfont(op.join(config['path'], path)))
             for path in directory.BIN]

    new_data = []
    for k in data:
        d = {'name': k}
        d.update(data[k])
        new_data.append(d)

    report_app = report_utils.BuildInfo(config)
    metrics = {'data': vmet._its_metrics, 'headings': vmet._its_metrics_header}
    table_sizes = {'tables': ttftablesizes[0], 'sizes': ttftablesizes[1:]}
    report_app.summary_page.dump_file(metrics, 'metrics.json')
    report_app.summary_page.dump_file(table_sizes, 'table_sizes.json')
    report_app.summary_page.dump_file(autohint_sizes, 'autohint_sizes.json')
    report_app.summary_page.dump_file(new_data, 'tests.json')
    report_app.summary_page.dump_file({'mean': ftables_data.mean,
                                       'grouped': ftables_data.grouped,
                                       'delta': ftables_data.delta},
                                      'fonts_tables_grouped.json')
    for face in family_metadata.fonts:
        face_template = "@font-face {{ font-family: {}; src: url(fonts/{});}}\n".format(face.metadata_object['postScriptName'], face.metadata_object['filename'])
        report_app.write_file(face_template, op.join(report_app.css_dir, 'faces.css'), mode='a')

    fonts_serialized = dict([(str(path), font_factory_instance_to_dict(fontaine)) for path, fontaine in fonts])
    report_app.summary_page.dump_file(fonts_serialized, 'fontaine_fonts.json')
    fonts_orthography = get_orthography(fonts)
    report_app.summary_page.dump_file({'fonts_list': fonts_orthography[0],
                                       'coverage_averages': fonts_orthography[1],
                                       'fonts_info': fonts_orthography[2]},
                                      'fonts_orthography.json')
<commit_msg>Get stem, related to issue #478<commit_after># coding: utf-8
# Copyright 2013 The Font Bakery Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# See AUTHORS.txt for the list of Authors and LICENSE.txt for the License.
from __future__ import print_function
from collections import defaultdict, Counter, namedtuple, OrderedDict
import os
import os.path as op
import yaml

import fontforge
from fontaine.cmap import Library
from fontaine.font import FontFactory

from bakery_cli.scripts.vmet import get_metric_view
from bakery_cli.utils import UpstreamDirectory

from bakery_cli.report import utils as report_utils

from bakery_lint.metadata import Metadata

TAB = 'Index'
TEMPLATE_DIR = op.join(op.dirname(__file__), 'templates')

t = lambda templatefile: op.join(TEMPLATE_DIR, templatefile)

def sort(data):
    a = []
    for grouped_dict in data:
        if 'required' in grouped_dict['tags']:
            a.append(grouped_dict)

    for grouped_dict in data:
        if 'note' in grouped_dict['tags'] and 'required' not in grouped_dict['tags']:
            a.append(grouped_dict)

    for grouped_dict in data:
        if 'note' not in grouped_dict['tags'] and 'required' not in grouped_dict['tags']:
            a.append(grouped_dict)

    return a

def filter_with_tag(fonttestdata, tag):
    tests = fonttestdata['failure'] + fonttestdata['error']
    return [test for test in tests if tag in test['tags']]

def filter_by_results_with_tag(fonttestdata, tag, *results):
    tests = {}
    for res in results:
        tests[res] = [test for test in fonttestdata.get(res) if tag in test.get('tags', [])]
    return tests

def get_fonts_table_sizes(fonts):
    """ Returns tuple with available tables from all fonts and their length """
    from fontTools.ttLib import sfnt
    _fonts = {}
    tables = []
    for font in fonts:
        _fonts[op.basename(font)] = {}
        with open(font) as fp_font:
            sf = sfnt.SFNTReader(fp_font)
            for t in sf.tables:
                if t not in tables:
                    tables.append(t)
                _fonts[op.basename(font)][t] = sf.tables[t].length
    return tables, _fonts

def get_fonts_table_sizes_grouped(fonts_list):
    _, fonts = get_fonts_table_sizes(fonts_list)
    fonts_dict = defaultdict(dict, fonts)

    # Fonts may have different tables!!!

    # across all fonts calculate sum of each table
    table_sizes_sums = sum(
        (Counter(v) for k, v in fonts_dict.iteritems()), Counter()
    )

    # count amount of each table across all fonts
    tables_counts = sum(
        (Counter(v.keys()) for k, v in fonts_dict.iteritems()), Counter()
    )

    # count average for each table, take value from 'table_sizes_sums'
    # and divide by corresponding value from  'tables_counts',
    # eg table_sizes_sums['glyf'] / tables_counts['glyf']
    tables_mean_dict = {
        k: table_sizes_sums[k]/tables_counts[k] for k in table_sizes_sums
    }

    # calculate deviation (delta) from an average
    # for each font and each table in font find delta
    tables_delta_dict = {}
    for font, tables in fonts_dict.iteritems():
        tables_delta_dict[font] = {
            k: tables_mean_dict[k]-v for k, v in tables.iteritems()
        }

    # gather all existent tables from all fonts
    all_possible_tables = set()
    for font, tables in tables_delta_dict.items():
        for table in tables:
            if table not in all_possible_tables:
                all_possible_tables.add(table)

    # if some font does not have a table that others have,
    # just set the deviation to 0
    for font, tables in tables_delta_dict.items():
        for item in all_possible_tables:
            tables.setdefault(item, 0)
        tables_delta_dict[font] = tables

    # make the deviation dict ready for google chart as array
    tables_delta_dict_for_google_array = {}
    for font, props in tables_delta_dict.iteritems():
        tables_delta_dict_for_google_array.setdefault('fonts', []).append(font)
        for k, v in props.iteritems():
            tables_delta_dict_for_google_array.setdefault(k, []).append(v)

    # prepare all tables dict as array for google chart
    tables_dict_for_google_array = {}
    for font, props in fonts.iteritems():
        tables_dict_for_google_array.setdefault('fonts', []).append(font)
        for k, v in props.iteritems():
            tables_dict_for_google_array.setdefault(k, []).append(v)

    grouped_dict = {
        'fonts': tables_dict_for_google_array.pop('fonts'),
        'tables': [
            [k, tables_mean_dict[k]] + v for k, v in tables_dict_for_google_array.items()
        ]
    }

    delta_dict = {
        'fonts': tables_delta_dict_for_google_array.pop('fonts'),
        'tables': [
            [k, ] + v for k, v in tables_delta_dict_for_google_array.items()
        ]
    }

    # make all arrays to have same len
    max_len = len(max(grouped_dict['tables'], key=len))
    new_items = []
    for item in grouped_dict["tables"]:
        new_item = item[:]
        while len(new_item) < max_len:
            new_item.append(0)
        new_items.append(new_item)
    grouped_dict["tables"] = new_items

    ftable = namedtuple('FontTable', ['mean', 'grouped', 'delta'])
    return ftable(tables_mean_dict, grouped_dict, delta_dict)

def get_orthography(fontaineFonts):
    fonts_dict = defaultdict(list)
    library = Library(collections=['subsets'])
    fonts_names = []
    for font, fontaine in fontaineFonts:
        fonts_names.append(font)
        for info in fontaine.get_orthographies(_library=library):
            font_info = dict(name=font, support=info.support_level,
                             coverage=info.coverage,
                             missing_chars=info.missing)
            fonts_dict[info.charmap.common_name].append(font_info)
    averages = {}
    for subset, fonts in fonts_dict.items():
        averages[subset] = sum([font['coverage'] for font in fonts]) / len(fonts)
    return sorted(fonts_names), averages, OrderedDict(sorted(fonts_dict.items()))

def to_google_data_list(tdict, haxis=0):
    return sorted([[x, tdict[x] - haxis] for x in tdict])

def font_table_to_google_data_list(tdict):
    return sorted([list(item) for item in tdict.items()])

def average_table_size(tdict):
    return sum(tdict.values()) / len(tdict)

def _obj_to_dict(instance, exclude_attrs=()):
    # Very simplified, but enough for reports
    return {
        k: getattr(instance, k) for k in dir(instance) \
        if not any([k.startswith('__'), str(k) in exclude_attrs])
    }

def font_factory_instance_to_dict(instance):
    return _obj_to_dict(instance, exclude_attrs=(
        'get_othography_info', 'get_orthographies', 'orthographies',
        'refresh_sfnt_properties', '_fontFace', 'getGlyphNames'
    ))

def get_stem(fontfile, glyph='n'):
    ttf = fontforge.open(fontfile)
    glyph = ttf[glyph]
    if not glyph.vhints:
        glyph.autoHint()
    if not glyph.vhints:
        return None
    v_hints = [item[1] for item in glyph.vhints]
    return sum(v_hints)/len(v_hints)

def generate(config):
    if config.get('failed'):
        return

    directory = UpstreamDirectory(config['path'])
    if op.exists(op.join(config['path'], 'METADATA.new.json')):
        metadata_file = open(op.join(config['path'], 'METADATA.new.json')).read()
    else:
        metadata_file = open(op.join(config['path'], 'METADATA.json')).read()
    family_metadata = Metadata.get_family_metadata(metadata_file)
    faces = []
    for f in family_metadata.fonts:
        faces.append({'name': f.full_name,
                      'basename': f.post_script_name,
                      'path': f.filename,
                      'meta': f})

    metadata = yaml.load(open(op.join(config['path'], 'METADATA.yaml')))
    upstreamdata = {}
    upstreamdatafile = op.join(config['path'], 'upstream.yaml')
    if op.exists(upstreamdatafile):
        upstreamdata = yaml.load(open(upstreamdatafile))

    data = {}
    for fp in directory.BIN:
        path = op.join(config['path'], '{}.yaml'.format(fp[:-4]))
        if op.exists(path):
            data[fp] = yaml.load(open(path))
    data.update(metadata)
    data.update(upstreamdata)

    fontpaths = [op.join(config['path'], path)
                 for path in directory.BIN]
    ttftablesizes = get_fonts_table_sizes(fontpaths)

    ftables_data = get_fonts_table_sizes_grouped(fontpaths)

    buildstate = yaml.load(open(op.join(config['path'], 'build.state.yaml')))
    autohint_sizes = buildstate.get('autohinting_sizes', [])
    vmet = get_metric_view(fontpaths)

    fonts = [(path, FontFactory.openfont(op.join(config['path'], path)))
             for path in directory.BIN]

    stems = [dict(fontname=path, stem=get_stem(op.join(config['path'], path))) for path in directory.BIN]

    new_data = []
    for k in data:
        d = {'name': k}
        d.update(data[k])
        new_data.append(d)

    report_app = report_utils.BuildInfo(config)
    metrics = {'data': vmet._its_metrics, 'headings': vmet._its_metrics_header}
    table_sizes = {'tables': ttftablesizes[0], 'sizes': ttftablesizes[1:]}
    report_app.summary_page.dump_file(metrics, 'metrics.json')
    report_app.summary_page.dump_file(stems, 'stems.json')
    report_app.summary_page.dump_file(table_sizes, 'table_sizes.json')
    report_app.summary_page.dump_file(autohint_sizes, 'autohint_sizes.json')
    report_app.summary_page.dump_file(new_data, 'tests.json')
    report_app.summary_page.dump_file({'mean': ftables_data.mean,
                                       'grouped': ftables_data.grouped,
                                       'delta': ftables_data.delta},
                                      'fonts_tables_grouped.json')
    for face in family_metadata.fonts:
        face_template = "@font-face {{ font-family: {}; src: url(fonts/{});}}\n".format(face.metadata_object['postScriptName'], face.metadata_object['filename'])
        report_app.write_file(face_template, op.join(report_app.css_dir, 'faces.css'), mode='a')

    fonts_serialized = dict([(str(path), font_factory_instance_to_dict(fontaine)) for path, fontaine in fonts])
    report_app.summary_page.dump_file(fonts_serialized, 'fontaine_fonts.json')
    fonts_orthography = get_orthography(fonts)
    report_app.summary_page.dump_file({'fonts_list': fonts_orthography[0],
                                       'coverage_averages': fonts_orthography[1],
                                       'fonts_info': fonts_orthography[2]},
                                      'fonts_orthography.json')
<|endoftext|>


<commit_before>from flask import Flask, render_template_string, Markup
from flask.ext.testing import TestCase, ContextVariableDoesNotExist
from flask.ext.cache import Cache
from flask.ext.twitter_oembedder import TwitterOEmbedder
import types
import httpretty

class FlaskStaticTest(TestCase):
    def create_app(self):
        app = Flask(__name__)
        app.config['TESTING']=True
        app.config['CACHE_TYPE'] = 'simple'
        app.config['TWITTER_CONSUMER_KEY'] = 'twitter_consumer_key'
        app.config['TWITTER_CONSUMER_SECRET'] = 'twitter_consumer_secret'
        app.config['TWITTER_ACCESS_TOKEN'] = 'twitter_access_token'
        app.config['TWITTER_TOKEN_SECRET'] = 'twitter_token_secret'
        self.cache = Cache(app)
        self.twitter_oembedder = TwitterOEmbedder(app,self.cache)

        @app.route('/')
        def index():
            return render_template_string('')

        return app

    def test_jinja_oembed_tweet_avaliable(self):
        response = self.client.get('/')
        assert type(self.get_context_variable('oembed_tweet')) is types.FunctionType

    @httpretty.activate
    def test_oembed_tweet_valid_id_debug_off(self):
        with open('data/99530515043983360.json') as f:
            httpretty.register_uri(httpretty.GET, 'https://api.twitter.com/1.1/statuses/oembed.json?id=99530515043983360',
                body = f.read())
        response = self.client.get('/')
        oembed_tweet = self.get_context_variable('oembed_tweet')
        valid = oembed_tweet('99530515043983360')
        assert type(valid) is Markup

    @httpretty.activate
    def test_oembed_tweet_invaild_id_debug_off(self):
        with open('data/abc.json') as f:
            httpretty.register_uri(httpretty.GET, 'https://api.twitter.com/1.1/statuses/oembed.json?id=abc',
                body = f.read())
        response = self.client.get('/')
        oembed_tweet = self.get_context_variable('oembed_tweet')
        invalid = oembed_tweet('abc')
        assert invalid is ''

    @httpretty.activate
    def test_oembed_tweet_invalid_id_debug_on(self):
        self.twitter_oembedder.init(self.app, self.cache, debug=True)
        with open('data/abc.json') as f:
            httpretty.register_uri(httpretty.GET, 'https://api.twitter.com/1.1/statuses/oembed.json?id=abc',
                body = f.read())
        response = self.client.get('/')
        oembed_tweet = self.get_context_variable('oembed_tweet')
        try:
            invalid = oembed_tweet('abc')
            assert False
        except Exception as e:
            assert type(e) is KeyError

<commit_msg>updated directory for jsons so tests can be run with 'python setup.py test'<commit_after>from flask import Flask, render_template_string, Markup
from flask.ext.testing import TestCase, ContextVariableDoesNotExist
from flask.ext.cache import Cache
from flask.ext.twitter_oembedder import TwitterOEmbedder
import types
import httpretty

class FlaskStaticTest(TestCase):
    def create_app(self):
        app = Flask(__name__)
        app.config['TESTING']=True
        app.config['CACHE_TYPE'] = 'simple'
        app.config['TWITTER_CONSUMER_KEY'] = 'twitter_consumer_key'
        app.config['TWITTER_CONSUMER_SECRET'] = 'twitter_consumer_secret'
        app.config['TWITTER_ACCESS_TOKEN'] = 'twitter_access_token'
        app.config['TWITTER_TOKEN_SECRET'] = 'twitter_token_secret'
        self.cache = Cache(app)
        self.twitter_oembedder = TwitterOEmbedder(app,self.cache)

        @app.route('/')
        def index():
            return render_template_string('')

        return app

    def test_jinja_oembed_tweet_avaliable(self):
        response = self.client.get('/')
        assert type(self.get_context_variable('oembed_tweet')) is types.FunctionType

    @httpretty.activate
    def test_oembed_tweet_valid_id_debug_off(self):
        with open('tests/data/99530515043983360.json') as f:
            httpretty.register_uri(httpretty.GET, 'https://api.twitter.com/1.1/statuses/oembed.json?id=99530515043983360',
                body = f.read())
        response = self.client.get('/')
        oembed_tweet = self.get_context_variable('oembed_tweet')
        valid = oembed_tweet('99530515043983360')
        assert type(valid) is Markup

    @httpretty.activate
    def test_oembed_tweet_invaild_id_debug_off(self):
        with open('tests/data/abc.json') as f:
            httpretty.register_uri(httpretty.GET, 'https://api.twitter.com/1.1/statuses/oembed.json?id=abc',
                body = f.read())
        response = self.client.get('/')
        oembed_tweet = self.get_context_variable('oembed_tweet')
        invalid = oembed_tweet('abc')
        assert invalid is ''

    @httpretty.activate
    def test_oembed_tweet_invalid_id_debug_on(self):
        self.twitter_oembedder.init(self.app, self.cache, debug=True)
        with open('tests/data/abc.json') as f:
            httpretty.register_uri(httpretty.GET, 'https://api.twitter.com/1.1/statuses/oembed.json?id=abc',
                body = f.read())
        response = self.client.get('/')
        oembed_tweet = self.get_context_variable('oembed_tweet')
        try:
            invalid = oembed_tweet('abc')
            assert False
        except Exception as e:
            assert type(e) is KeyError

<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

from __future__ import division
from django.conf import settings
from django.core.exceptions import ObjectDoesNotExist
from django.utils.timezone import localtime
from ballot_box.models import *
from ballot_box.utils_data import Framer, Checker
import logging
import time
import datetime
import os.path
import shutil
import pytz

logger = logging.getLogger("kpcc_backroom_handshakes")

checker = Checker()

class Saver(object):
    """
    """

    log_message = "\n*** My Import Messages ***\n"

    def make_office(self, office):
        """
        """
        log_message = ""
        try:
            obj, created = Office.objects.update_or_create(
                officeid=office["officeid"],
                defaults={
                    "name": office["officename"],
                    "slug": office["officeslug"],
                    "active": office["active"],
                    "poss_error": False,
                }
            )
            if created:
                log_message += "* %s created\n" % (office["officeslug"])
            else:
                log_message += "* %s exists\n" % (office["officeslug"])
        except Exception, exception:
            error_output = "%s %s" % (exception, office["officeslug"])
            logger.error(error_output)
            raise
        return log_message

    def make_contest(self, office, contest):
        """
        """
        log_message = ""
        try:
            this_office = Office.objects.get(officeid=office["officeid"])
        except Exception, exception:
            error_output = "%s %s" % (exception, office["officeid"])
            logger.error(error_output)
            raise
        try:
            obj, created = this_office.contest_set.update_or_create(
                election_id=contest["election_id"],
                resultsource_id=contest["resultsource_id"],
                contestid=contest["contestid"],
                defaults={
                    "contestname": contest["contestname"],
                    "seatnum": contest["seatnum"],
                    "contestdescription": contest["contestdescription"],
                    "is_uncontested": contest["is_uncontested"],
                    "is_national": contest["is_national"],
                    "is_statewide": contest["is_statewide"],
                    "is_ballot_measure": contest["is_ballot_measure"],
                    "is_judicial": contest["is_judicial"],
                    "reporttype": contest["reporttype"],
                    "precinctstotal": contest["precinctstotal"],
                    "precinctsreporting": contest["precinctsreporting"],
                    "precinctsreportingpct": contest["precinctsreportingpct"],
                    "votersregistered": contest["votersregistered"],
                    "votersturnout": contest["votersturnout"],
                    "poss_error": contest["poss_error"],
                }
            )
            if created:
                log_message += "\t* %s created\n" % (contest["contestid"])
            else:
                log_message += "\t* %s exists but we updated figures\n" % (contest["contestid"])
        except Exception, exception:
            error_output = "%s %s" % (exception, contest["contestid"])
            logger.error(error_output)
            raise
        return log_message

    def make_judicial(self, contest, judicial):
        """
        """
        log_message = ""
        try:
            this_contest = Contest.objects.get(contestid=contest["contestid"])
        except Exception, exception:
            error_output = "%s %s" % (exception, contest["contestid"])
            logger.error(error_output)
            raise
        try:
            obj, created = this_contest.judicialcandidate_set.update_or_create(
                judgeid=judicial["judgeid"],
                defaults={
                    "ballotorder": judicial["ballotorder"],
                    "firstname": judicial["firstname"],
                    "lastname": judicial["lastname"],
                    "fullname": judicial["fullname"],
                    "yescount": judicial["yescount"],
                    "yespct": judicial["yespct"] / 100,
                    "nocount": judicial["nocount"],
                    "nopct": judicial["nopct"] / 100,
                    "poss_error": judicial["poss_error"],
                }
            )
            if created:
                log_message += "\t\t* %s created\n" % (judicial["judgeid"])
            else:
                log_message += "\t\t* %s exists but we updated figures\n" % (judicial["judgeid"])
        except Exception, exception:
            error_output = "%s %s" % (exception, judicial["judgeid"])
            logger.error(error_output)
            raise
        return log_message

    def make_measure(self, contest, measure):
        """
        """
        log_message = ""
        try:
            this_contest = Contest.objects.get(contestid=contest["contestid"])
        except Exception, exception:
            error_output = "%s %s" % (exception, contest["contestid"])
            logger.error(error_output)
            raise
        try:
            obj, created = this_contest.ballotmeasure_set.update_or_create(
                measureid=measure["measureid"],
                defaults={
                    "ballotorder": measure["ballotorder"],
                    "fullname": measure["fullname"],
                    "description": measure["description"],
                    "yescount": measure["yescount"],
                    "yespct": measure["yespct"] / 100,
                    "nocount": measure["nocount"],
                    "nopct": measure["nopct"] / 100,
                    "poss_error": measure["poss_error"],
                }
            )
            if created:
                log_message += "\t\t* %s created\n" % (measure["measureid"])
            else:
                log_message += "\t\t* %s exists but we updated figures\n" % (measure["measureid"])
        except Exception, exception:
            error_output = "%s %s" % (exception, measure["measureid"])
            logger.error(error_output)
            raise
        return log_message

    def make_candidate(self, contest, candidate):
        """
        """
        log_message = ""
        try:
            this_contest = Contest.objects.get(contestid=contest["contestid"])
        except Exception, exception:
            error_output = "%s %s" % (exception, contest["contestid"])
            logger.error(error_output)
            raise
        try:
            obj, created = this_contest.candidate_set.update_or_create(
                candidateid=candidate["candidateid"],
                defaults={
                    "ballotorder": candidate["ballotorder"],
                    "firstname": candidate["firstname"],
                    "lastname": candidate["lastname"],
                    "fullname": candidate["fullname"],
                    "party": candidate["party"],
                    "incumbent": candidate["incumbent"],
                    "votecount": candidate["votecount"],
                    "votepct": candidate["votepct"],
                    "poss_error": candidate["poss_error"],
                }
            )
            if created:
                log_message += "\t\t* %s created\n" % (candidate["candidateid"])
            else:
                log_message += "\t\t* %s exists but we updated figures\n" % (candidate["candidateid"])
        except Exception, exception:
            error_output = "%s %s" % (exception, candidate["candidateid"])
            logger.error(error_output)
            raise
        return log_message

    def _eval_timestamps(self, file_time, database_time):
        """
        """
        fttz = localtime(file_time).tzinfo
        dbtz = localtime(database_time).tzinfo
        if fttz == None and dbtz == None:
            raise Exception
        elif fttz == None:
            raise Exception
        elif dbtz == None:
            raise Exception
        else:
            if fttz._tzname == "PDT" or fttz._tzname == "PST":
                if file_time > database_time:
                    return True
                else:
                    return False
            else:
                raise Exception

    def _update_result_timestamps(self, src, file_timestamp):
        """
        """
        obj = ResultSource.objects.get(source_slug=src.source_slug)
        obj.source_latest = file_timestamp
        obj.save(update_fields=["source_latest"])

    def _make_office_id(self, *args, **kwargs):
        """
        """
        framer = Framer()
        required_keys = [
            "source_short",
            "officeslug",
        ]
        if len(args) != len(required_keys):
            raise Exception
        else:
            pass
        if kwargs:
            args = list(args)
            # this needs work but it's a start
            args.append(kwargs.values()[0].lower())
            output = framer._concat(*args, delimiter="-")
        else:
            output = framer._concat(*args, delimiter="-")
        return output

    def _make_contest_id(self, *args, **kwargs):
        """
        """
        framer = Framer()
        required_keys = [
            "electionid",
            "source_short",
            "level",
            "officeslug",
        ]
        if len(args) != len(required_keys):
            raise Exception
        else:
            pass
        if kwargs:
            args = list(args)
            # this needs work but it's a start
            args.append(str(kwargs.values()[0]).lower())
            output = framer._concat(*args, delimiter="-")
        else:
            output = framer._concat(*args, delimiter="-")
        return output

    def _make_this_id(self, contest_type, *args, **kwargs):
        """
        """
        framer = Framer()
        if contest_type == "measure":
            required_keys = [
                "contestid",
                "measure_id",
            ]
        elif contest_type == "judicial":
            required_keys = [
                "contestid",
                "judicialslug",
            ]
        elif contest_type == "candidate":
            required_keys = [
                "contestid",
                "candidateslug",
            ]
        if len(args) != len(required_keys):
            raise Exception
        else:
            pass
        if kwargs:
            args = list(args)
            # this needs work but it's a start
            args.append(str(kwargs.values()[0]).lower())
            output = framer._concat(*args, delimiter="-")
        else:
            output = framer._concat(*args, delimiter="-")
        return output
<commit_msg>updates to importer to make sure we're converting to decimal on import where needed<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

from __future__ import division
from django.conf import settings
from django.core.exceptions import ObjectDoesNotExist
from django.utils.timezone import localtime
from ballot_box.models import *
from ballot_box.utils_data import Framer, Checker
import logging
import time
import datetime
import os.path
import shutil
import pytz

logger = logging.getLogger("kpcc_backroom_handshakes")

checker = Checker()

class Saver(object):
    """
    """

    log_message = "\n*** My Import Messages ***\n"

    def make_office(self, office):
        """
        """
        log_message = ""
        try:
            obj, created = Office.objects.update_or_create(
                officeid=office["officeid"],
                defaults={
                    "name": office["officename"],
                    "slug": office["officeslug"],
                    "active": office["active"],
                    "poss_error": False,
                }
            )
            if created:
                log_message += "* %s created\n" % (office["officeslug"])
            else:
                log_message += "* %s exists\n" % (office["officeslug"])
        except Exception, exception:
            error_output = "%s %s" % (exception, office["officeslug"])
            logger.error(error_output)
            raise
        return log_message

    def make_contest(self, office, contest):
        """
        """
        log_message = ""
        try:
            this_office = Office.objects.get(officeid=office["officeid"])
        except Exception, exception:
            error_output = "%s %s" % (exception, office["officeid"])
            logger.error(error_output)
            raise
        try:
            obj, created = this_office.contest_set.update_or_create(
                election_id=contest["election_id"],
                resultsource_id=contest["resultsource_id"],
                contestid=contest["contestid"],
                defaults={
                    "contestname": contest["contestname"],
                    "seatnum": contest["seatnum"],
                    "contestdescription": contest["contestdescription"],
                    "is_uncontested": contest["is_uncontested"],
                    "is_national": contest["is_national"],
                    "is_statewide": contest["is_statewide"],
                    "is_ballot_measure": contest["is_ballot_measure"],
                    "is_judicial": contest["is_judicial"],
                    "reporttype": contest["reporttype"],
                    "precinctstotal": contest["precinctstotal"],
                    "precinctsreporting": contest["precinctsreporting"],
                    "precinctsreportingpct": contest["precinctsreportingpct"] / 100,
                    "votersregistered": contest["votersregistered"],
                    "votersturnout": contest["votersturnout"],
                    "poss_error": contest["poss_error"],
                }
            )
            if created:
                log_message += "\t* %s created\n" % (contest["contestid"])
            else:
                log_message += "\t* %s exists but we updated figures\n" % (contest["contestid"])
        except Exception, exception:
            error_output = "%s %s" % (exception, contest["contestid"])
            logger.error(error_output)
            raise
        return log_message

    def make_judicial(self, contest, judicial):
        """
        """
        log_message = ""
        try:
            this_contest = Contest.objects.get(contestid=contest["contestid"])
        except Exception, exception:
            error_output = "%s %s" % (exception, contest["contestid"])
            logger.error(error_output)
            raise
        try:
            obj, created = this_contest.judicialcandidate_set.update_or_create(
                judgeid=judicial["judgeid"],
                defaults={
                    "ballotorder": judicial["ballotorder"],
                    "firstname": judicial["firstname"],
                    "lastname": judicial["lastname"],
                    "fullname": judicial["fullname"],
                    "yescount": judicial["yescount"],
                    "yespct": judicial["yespct"] / 100,
                    "nocount": judicial["nocount"],
                    "nopct": judicial["nopct"] / 100,
                    "poss_error": judicial["poss_error"],
                }
            )
            if created:
                log_message += "\t\t* %s created\n" % (judicial["judgeid"])
            else:
                log_message += "\t\t* %s exists but we updated figures\n" % (judicial["judgeid"])
        except Exception, exception:
            error_output = "%s %s" % (exception, judicial["judgeid"])
            logger.error(error_output)
            raise
        return log_message

    def make_measure(self, contest, measure):
        """
        """
        log_message = ""
        try:
            this_contest = Contest.objects.get(contestid=contest["contestid"])
        except Exception, exception:
            error_output = "%s %s" % (exception, contest["contestid"])
            logger.error(error_output)
            raise
        try:
            obj, created = this_contest.ballotmeasure_set.update_or_create(
                measureid=measure["measureid"],
                defaults={
                    "ballotorder": measure["ballotorder"],
                    "fullname": measure["fullname"],
                    "description": measure["description"],
                    "yescount": measure["yescount"],
                    "yespct": measure["yespct"] / 100,
                    "nocount": measure["nocount"],
                    "nopct": measure["nopct"] / 100,
                    "poss_error": measure["poss_error"],
                }
            )
            if created:
                log_message += "\t\t* %s created\n" % (measure["measureid"])
            else:
                log_message += "\t\t* %s exists but we updated figures\n" % (measure["measureid"])
        except Exception, exception:
            error_output = "%s %s" % (exception, measure["measureid"])
            logger.error(error_output)
            raise
        return log_message

    def make_candidate(self, contest, candidate):
        """
        """
        log_message = ""
        try:
            this_contest = Contest.objects.get(contestid=contest["contestid"])
        except Exception, exception:
            error_output = "%s %s" % (exception, contest["contestid"])
            logger.error(error_output)
            raise
        try:
            obj, created = this_contest.candidate_set.update_or_create(
                candidateid=candidate["candidateid"],
                defaults={
                    "ballotorder": candidate["ballotorder"],
                    "firstname": candidate["firstname"],
                    "lastname": candidate["lastname"],
                    "fullname": candidate["fullname"],
                    "party": candidate["party"],
                    "incumbent": candidate["incumbent"],
                    "votecount": candidate["votecount"],
                    "votepct": candidate["votepct"] / 100,
                    "poss_error": candidate["poss_error"],
                }
            )
            if created:
                log_message += "\t\t* %s created\n" % (candidate["candidateid"])
            else:
                log_message += "\t\t* %s exists but we updated figures\n" % (candidate["candidateid"])
        except Exception, exception:
            error_output = "%s %s" % (exception, candidate["candidateid"])
            logger.error(error_output)
            raise
        return log_message

    def _eval_timestamps(self, file_time, database_time):
        """
        """
        fttz = localtime(file_time).tzinfo
        dbtz = localtime(database_time).tzinfo
        if fttz == None and dbtz == None:
            raise Exception
        elif fttz == None:
            raise Exception
        elif dbtz == None:
            raise Exception
        else:
            if fttz._tzname == "PDT" or fttz._tzname == "PST":
                if file_time > database_time:
                    return True
                else:
                    return False
            else:
                raise Exception

    def _update_result_timestamps(self, src, file_timestamp):
        """
        """
        obj = ResultSource.objects.get(source_slug=src.source_slug)
        obj.source_latest = file_timestamp
        obj.save(update_fields=["source_latest"])

    def _make_office_id(self, *args, **kwargs):
        """
        """
        framer = Framer()
        required_keys = [
            "source_short",
            "officeslug",
        ]
        if len(args) != len(required_keys):
            raise Exception
        else:
            pass
        if kwargs:
            args = list(args)
            # this needs work but it's a start
            args.append(kwargs.values()[0].lower())
            output = framer._concat(*args, delimiter="-")
        else:
            output = framer._concat(*args, delimiter="-")
        return output

    def _make_contest_id(self, *args, **kwargs):
        """
        """
        framer = Framer()
        required_keys = [
            "electionid",
            "source_short",
            "level",
            "officeslug",
        ]
        if len(args) != len(required_keys):
            raise Exception
        else:
            pass
        if kwargs:
            args = list(args)
            # this needs work but it's a start
            args.append(str(kwargs.values()[0]).lower())
            output = framer._concat(*args, delimiter="-")
        else:
            output = framer._concat(*args, delimiter="-")
        return output

    def _make_this_id(self, contest_type, *args, **kwargs):
        """
        """
        framer = Framer()
        if contest_type == "measure":
            required_keys = [
                "contestid",
                "measure_id",
            ]
        elif contest_type == "judicial":
            required_keys = [
                "contestid",
                "judicialslug",
            ]
        elif contest_type == "candidate":
            required_keys = [
                "contestid",
                "candidateslug",
            ]
        if len(args) != len(required_keys):
            raise Exception
        else:
            pass
        if kwargs:
            args = list(args)
            # this needs work but it's a start
            args.append(str(kwargs.values()[0]).lower())
            output = framer._concat(*args, delimiter="-")
        else:
            output = framer._concat(*args, delimiter="-")
        return output
<|endoftext|>