<commit_before># -*- coding: utf-8 -*-
# Copyright 2014-2016 OpenMarket Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from synapse.api.errors import (
    cs_exception, SynapseError, CodeMessageException, UnrecognizedRequestError, Codes
)
from synapse.util.logcontext import LoggingContext, PreserveLoggingContext
from synapse.util.caches import intern_dict
from synapse.util.metrics import Measure
import synapse.metrics
import synapse.events

from canonicaljson import (
    encode_canonical_json, encode_pretty_printed_json
)

from twisted.internet import defer
from twisted.web import server, resource
from twisted.web.server import NOT_DONE_YET
from twisted.web.util import redirectTo

import collections
import logging
import urllib
import ujson

logger = logging.getLogger(__name__)

metrics = synapse.metrics.get_metrics_for(__name__)

incoming_requests_counter = metrics.register_counter(
    "requests",
    labels=["method", "servlet", "tag"],
)
outgoing_responses_counter = metrics.register_counter(
    "responses",
    labels=["method", "code"],
)

response_timer = metrics.register_distribution(
    "response_time",
    labels=["method", "servlet", "tag"]
)

response_ru_utime = metrics.register_distribution(
    "response_ru_utime", labels=["method", "servlet", "tag"]
)

response_ru_stime = metrics.register_distribution(
    "response_ru_stime", labels=["method", "servlet", "tag"]
)

response_db_txn_count = metrics.register_distribution(
    "response_db_txn_count", labels=["method", "servlet", "tag"]
)

response_db_txn_duration = metrics.register_distribution(
    "response_db_txn_duration", labels=["method", "servlet", "tag"]
)

_next_request_id = 0

def request_handler(report_metrics=True):
    """Decorator for ``wrap_request_handler``"""
    return lambda request_handler: wrap_request_handler(request_handler, report_metrics)

def wrap_request_handler(request_handler, report_metrics):
    """Wraps a method that acts as a request handler with the necessary logging
    and exception handling.

    The method must have a signature of "handle_foo(self, request)". The
    argument "self" must have "version_string" and "clock" attributes. The
    argument "request" must be a twisted HTTP request.

    The method must return a deferred. If the deferred succeeds we assume that
    a response has been sent. If the deferred fails with a SynapseError we use
    it to send a JSON response with the appropriate HTTP reponse code. If the
    deferred fails with any other type of error we send a 500 reponse.

    We insert a unique request-id into the logging context for this request and
    log the response and duration for this request.
    """

    @defer.inlineCallbacks
    def wrapped_request_handler(self, request):
        global _next_request_id
        request_id = "%s-%s" % (request.method, _next_request_id)
        _next_request_id += 1

        with LoggingContext(request_id) as request_context:
            if report_metrics:
                request_metrics = RequestMetrics()
                request_metrics.start(self.clock)

            request_context.request = request_id
            with request.processing():
                try:
                    with PreserveLoggingContext(request_context):
                        yield request_handler(self, request)
                except CodeMessageException as e:
                    code = e.code
                    if isinstance(e, SynapseError):
                        logger.info(
                            "%s SynapseError: %s - %s", request, code, e.msg
                        )
                    else:
                        logger.exception(e)
                    outgoing_responses_counter.inc(request.method, str(code))
                    respond_with_json(
                        request, code, cs_exception(e), send_cors=True,
                        pretty_print=_request_user_agent_is_curl(request),
                        version_string=self.version_string,
                    )
                except:
                    logger.exception(
                        "Failed handle request %s.%s on %r: %r",
                        request_handler.__module__,
                        request_handler.__name__,
                        self,
                        request
                    )
                    respond_with_json(
                        request,
                        500,
                        {
                            "error": "Internal server error",
                            "errcode": Codes.UNKNOWN,
                        },
                        send_cors=True
                    )
                finally:
                    try:
                        if report_metrics:
                            request_metrics.stop(
                                self.clock, request, self.__class__.__name__
                            )
                    except:
                        pass
    return wrapped_request_handler

class HttpServer(object):
    """ Interface for registering callbacks on a HTTP server
    """

    def register_paths(self, method, path_patterns, callback):
        """ Register a callback that gets fired if we receive a http request
        with the given method for a path that matches the given regex.

        If the regex contains groups these gets passed to the calback via
        an unpacked tuple.

        Args:
            method (str): The method to listen to.
            path_patterns (list<SRE_Pattern>): The regex used to match requests.
            callback (function): The function to fire if we receive a matched
                request. The first argument will be the request object and
                subsequent arguments will be any matched groups from the regex.
                This should return a tuple of (code, response).
        """
        pass

class JsonResource(HttpServer, resource.Resource):
    """ This implements the HttpServer interface and provides JSON support for
    Resources.

    Register callbacks via register_path()

    Callbacks can return a tuple of status code and a dict in which case the
    the dict will automatically be sent to the client as a JSON object.

    The JsonResource is primarily intended for returning JSON, but callbacks
    may send something other than JSON, they may do so by using the methods
    on the request object and instead returning None.
    """

    isLeaf = True

    _PathEntry = collections.namedtuple("_PathEntry", ["pattern", "callback"])

    def __init__(self, hs, canonical_json=True):
        resource.Resource.__init__(self)

        self.canonical_json = canonical_json
        self.clock = hs.get_clock()
        self.path_regexs = {}
        self.version_string = hs.version_string
        self.hs = hs

    def register_paths(self, method, path_patterns, callback):
        for path_pattern in path_patterns:
            logger.debug("Registering for %s %s", method, path_pattern.pattern)
            self.path_regexs.setdefault(method, []).append(
                self._PathEntry(path_pattern, callback)
            )

    def render(self, request):
        """ This gets called by twisted every time someone sends us a request.
        """
        self._async_render(request)
        return server.NOT_DONE_YET

    # Disable metric reporting because _async_render does its own metrics.
    # It does its own metric reporting because _async_render dispatches to
    # a callback and it's the class name of that callback we want to report
    # against rather than the JsonResource itself.
    @request_handler(report_metrics=False)
    @defer.inlineCallbacks
    def _async_render(self, request):
        """ This gets called from render() every time someone sends us a request.
            This checks if anyone has registered a callback for that method and
            path.
        """
        if request.method == "OPTIONS":
            self._send_response(request, 200, {})
            return

        request_metrics = RequestMetrics()
        request_metrics.start(self.clock)

        with Measure(self.clock, "http.render"):
            # Loop through all the registered callbacks to check if the method
            # and path regex match
            for path_entry in self.path_regexs.get(request.method, []):
                m = path_entry.pattern.match(request.path)
                if not m:
                    continue

                # We found a match! Trigger callback and then return the
                # returned response. We pass both the request and any
                # matched groups from the regex to the callback.

                callback = path_entry.callback

                servlet_instance = getattr(callback, "__self__", None)
                if servlet_instance is not None:
                    servlet_classname = servlet_instance.__class__.__name__
                else:
                    servlet_classname = "%r" % callback

                kwargs = intern_dict({
                    name: urllib.unquote(value).decode("UTF-8") if value else value
                    for name, value in m.groupdict().items()
                })

                callback_return = yield callback(request, **kwargs)
                if callback_return is not None:
                    code, response = callback_return
                    self._send_response(request, code, response)

                try:
                    request_metrics.stop(self.clock, request, servlet_classname)
                except:
                    pass

                return

        # Huh. No one wanted to handle that? Fiiiiiine. Send 400.
        raise UnrecognizedRequestError()

    def _send_response(self, request, code, response_json_object,
                       response_code_message=None):
        # could alternatively use request.notifyFinish() and flip a flag when
        # the Deferred fires, but since the flag is RIGHT THERE it seems like
        # a waste.
        if request._disconnected:
            logger.warn(
                "Not sending response to request %s, already disconnected.",
                request)
            return

        outgoing_responses_counter.inc(request.method, str(code))

        # TODO: Only enable CORS for the requests that need it.
        respond_with_json(
            request, code, response_json_object,
            send_cors=True,
            response_code_message=response_code_message,
            pretty_print=_request_user_agent_is_curl(request),
            version_string=self.version_string,
            canonical_json=self.canonical_json,
        )

class RequestMetrics(object):
    def start(self, clock):
        self.start = clock.time_msec()
        self.start_context = LoggingContext.current_context()

    def stop(self, clock, request, servlet_classname):
        context = LoggingContext.current_context()

        tag = ""
        if context:
            tag = context.tag

            if context != self.start_context:
                logger.warn(
                    "Context have unexpectedly changed %r, %r",
                    context, self.start_context
                )
                return

        incoming_requests_counter.inc(request.method, servlet_classname, tag)

        response_timer.inc_by(
            clock.time_msec() - self.start, request.method,
            servlet_classname, tag
        )

        ru_utime, ru_stime = context.get_resource_usage()

        response_ru_utime.inc_by(
            ru_utime, request.method, servlet_classname, tag
        )
        response_ru_stime.inc_by(
            ru_stime, request.method, servlet_classname, tag
        )
        response_db_txn_count.inc_by(
            context.db_txn_count, request.method, servlet_classname, tag
        )
        response_db_txn_duration.inc_by(
            context.db_txn_duration, request.method, servlet_classname, tag
        )

class RootRedirect(resource.Resource):
    """Redirects the root '/' path to another path."""

    def __init__(self, path):
        resource.Resource.__init__(self)
        self.url = path

    def render_GET(self, request):
        return redirectTo(self.url, request)

    def getChild(self, name, request):
        if len(name) == 0:
            return self  # select ourselves as the child to render
        return resource.Resource.getChild(self, name, request)

def respond_with_json(request, code, json_object, send_cors=False,
                      response_code_message=None, pretty_print=False,
                      version_string="", canonical_json=True):
    if pretty_print:
        json_bytes = encode_pretty_printed_json(json_object) + "\n"
    else:
        if canonical_json or synapse.events.USE_FROZEN_DICTS:
            json_bytes = encode_canonical_json(json_object)
        else:
            # ujson doesn't like frozen_dicts.
            json_bytes = ujson.dumps(json_object, ensure_ascii=False)

    return respond_with_json_bytes(
        request, code, json_bytes,
        send_cors=send_cors,
        response_code_message=response_code_message,
        version_string=version_string
    )

def respond_with_json_bytes(request, code, json_bytes, send_cors=False,
                            version_string="", response_code_message=None):
    """Sends encoded JSON in response to the given request.

    Args:
        request (twisted.web.http.Request): The http request to respond to.
        code (int): The HTTP response code.
        json_bytes (bytes): The json bytes to use as the response body.
        send_cors (bool): Whether to send Cross-Origin Resource Sharing headers
            http://www.w3.org/TR/cors/
    Returns:
        twisted.web.server.NOT_DONE_YET"""

    request.setResponseCode(code, message=response_code_message)
    request.setHeader(b"Content-Type", b"application/json")
    request.setHeader(b"Server", version_string)
    request.setHeader(b"Content-Length", b"%d" % (len(json_bytes),))

    if send_cors:
        request.setHeader("Access-Control-Allow-Origin", "*")
        request.setHeader("Access-Control-Allow-Methods",
                          "GET, POST, PUT, DELETE, OPTIONS")
        request.setHeader("Access-Control-Allow-Headers",
                          "Origin, X-Requested-With, Content-Type, Accept")

    request.write(json_bytes)
    finish_request(request)
    return NOT_DONE_YET

def finish_request(request):
    """ Finish writing the response to the request.

    Twisted throws a RuntimeException if the connection closed before the
    response was written but doesn't provide a convenient or reliable way to
    determine if the connection was closed. So we catch and log the RuntimeException

    You might think that ``request.notifyFinish`` could be used to tell if the
    request was finished. However the deferred it returns won't fire if the
    connection was already closed, meaning we'd have to have called the method
    right at the start of the request. By the time we want to write the response
    it will already be too late.
    """
    try:
        request.finish()
    except RuntimeError as e:
        logger.info("Connection disconnected before response was written: %r", e)

def _request_user_agent_is_curl(request):
    user_agents = request.requestHeaders.getRawHeaders(
        "User-Agent", default=[]
    )
    for user_agent in user_agents:
        if "curl" in user_agent:
            return True
    return False
<commit_msg>Remove redundant measure<commit_after># -*- coding: utf-8 -*-
# Copyright 2014-2016 OpenMarket Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from synapse.api.errors import (
    cs_exception, SynapseError, CodeMessageException, UnrecognizedRequestError, Codes
)
from synapse.util.logcontext import LoggingContext, PreserveLoggingContext
from synapse.util.caches import intern_dict
from synapse.util.metrics import Measure
import synapse.metrics
import synapse.events

from canonicaljson import (
    encode_canonical_json, encode_pretty_printed_json
)

from twisted.internet import defer
from twisted.web import server, resource
from twisted.web.server import NOT_DONE_YET
from twisted.web.util import redirectTo

import collections
import logging
import urllib
import ujson

logger = logging.getLogger(__name__)

metrics = synapse.metrics.get_metrics_for(__name__)

incoming_requests_counter = metrics.register_counter(
    "requests",
    labels=["method", "servlet", "tag"],
)
outgoing_responses_counter = metrics.register_counter(
    "responses",
    labels=["method", "code"],
)

response_timer = metrics.register_distribution(
    "response_time",
    labels=["method", "servlet", "tag"]
)

response_ru_utime = metrics.register_distribution(
    "response_ru_utime", labels=["method", "servlet", "tag"]
)

response_ru_stime = metrics.register_distribution(
    "response_ru_stime", labels=["method", "servlet", "tag"]
)

response_db_txn_count = metrics.register_distribution(
    "response_db_txn_count", labels=["method", "servlet", "tag"]
)

response_db_txn_duration = metrics.register_distribution(
    "response_db_txn_duration", labels=["method", "servlet", "tag"]
)

_next_request_id = 0

def request_handler(report_metrics=True):
    """Decorator for ``wrap_request_handler``"""
    return lambda request_handler: wrap_request_handler(request_handler, report_metrics)

def wrap_request_handler(request_handler, report_metrics):
    """Wraps a method that acts as a request handler with the necessary logging
    and exception handling.

    The method must have a signature of "handle_foo(self, request)". The
    argument "self" must have "version_string" and "clock" attributes. The
    argument "request" must be a twisted HTTP request.

    The method must return a deferred. If the deferred succeeds we assume that
    a response has been sent. If the deferred fails with a SynapseError we use
    it to send a JSON response with the appropriate HTTP reponse code. If the
    deferred fails with any other type of error we send a 500 reponse.

    We insert a unique request-id into the logging context for this request and
    log the response and duration for this request.
    """

    @defer.inlineCallbacks
    def wrapped_request_handler(self, request):
        global _next_request_id
        request_id = "%s-%s" % (request.method, _next_request_id)
        _next_request_id += 1

        with LoggingContext(request_id) as request_context:
            if report_metrics:
                request_metrics = RequestMetrics()
                request_metrics.start(self.clock)

            request_context.request = request_id
            with request.processing():
                try:
                    with PreserveLoggingContext(request_context):
                        yield request_handler(self, request)
                except CodeMessageException as e:
                    code = e.code
                    if isinstance(e, SynapseError):
                        logger.info(
                            "%s SynapseError: %s - %s", request, code, e.msg
                        )
                    else:
                        logger.exception(e)
                    outgoing_responses_counter.inc(request.method, str(code))
                    respond_with_json(
                        request, code, cs_exception(e), send_cors=True,
                        pretty_print=_request_user_agent_is_curl(request),
                        version_string=self.version_string,
                    )
                except:
                    logger.exception(
                        "Failed handle request %s.%s on %r: %r",
                        request_handler.__module__,
                        request_handler.__name__,
                        self,
                        request
                    )
                    respond_with_json(
                        request,
                        500,
                        {
                            "error": "Internal server error",
                            "errcode": Codes.UNKNOWN,
                        },
                        send_cors=True
                    )
                finally:
                    try:
                        if report_metrics:
                            request_metrics.stop(
                                self.clock, request, self.__class__.__name__
                            )
                    except:
                        pass
    return wrapped_request_handler

class HttpServer(object):
    """ Interface for registering callbacks on a HTTP server
    """

    def register_paths(self, method, path_patterns, callback):
        """ Register a callback that gets fired if we receive a http request
        with the given method for a path that matches the given regex.

        If the regex contains groups these gets passed to the calback via
        an unpacked tuple.

        Args:
            method (str): The method to listen to.
            path_patterns (list<SRE_Pattern>): The regex used to match requests.
            callback (function): The function to fire if we receive a matched
                request. The first argument will be the request object and
                subsequent arguments will be any matched groups from the regex.
                This should return a tuple of (code, response).
        """
        pass

class JsonResource(HttpServer, resource.Resource):
    """ This implements the HttpServer interface and provides JSON support for
    Resources.

    Register callbacks via register_path()

    Callbacks can return a tuple of status code and a dict in which case the
    the dict will automatically be sent to the client as a JSON object.

    The JsonResource is primarily intended for returning JSON, but callbacks
    may send something other than JSON, they may do so by using the methods
    on the request object and instead returning None.
    """

    isLeaf = True

    _PathEntry = collections.namedtuple("_PathEntry", ["pattern", "callback"])

    def __init__(self, hs, canonical_json=True):
        resource.Resource.__init__(self)

        self.canonical_json = canonical_json
        self.clock = hs.get_clock()
        self.path_regexs = {}
        self.version_string = hs.version_string
        self.hs = hs

    def register_paths(self, method, path_patterns, callback):
        for path_pattern in path_patterns:
            logger.debug("Registering for %s %s", method, path_pattern.pattern)
            self.path_regexs.setdefault(method, []).append(
                self._PathEntry(path_pattern, callback)
            )

    def render(self, request):
        """ This gets called by twisted every time someone sends us a request.
        """
        self._async_render(request)
        return server.NOT_DONE_YET

    # Disable metric reporting because _async_render does its own metrics.
    # It does its own metric reporting because _async_render dispatches to
    # a callback and it's the class name of that callback we want to report
    # against rather than the JsonResource itself.
    @request_handler(report_metrics=False)
    @defer.inlineCallbacks
    def _async_render(self, request):
        """ This gets called from render() every time someone sends us a request.
            This checks if anyone has registered a callback for that method and
            path.
        """
        if request.method == "OPTIONS":
            self._send_response(request, 200, {})
            return

        request_metrics = RequestMetrics()
        request_metrics.start(self.clock)

        # Loop through all the registered callbacks to check if the method
        # and path regex match
        for path_entry in self.path_regexs.get(request.method, []):
            m = path_entry.pattern.match(request.path)
            if not m:
                continue

            # We found a match! Trigger callback and then return the
            # returned response. We pass both the request and any
            # matched groups from the regex to the callback.

            callback = path_entry.callback

            servlet_instance = getattr(callback, "__self__", None)
            if servlet_instance is not None:
                servlet_classname = servlet_instance.__class__.__name__
            else:
                servlet_classname = "%r" % callback

            kwargs = intern_dict({
                name: urllib.unquote(value).decode("UTF-8") if value else value
                for name, value in m.groupdict().items()
            })

            callback_return = yield callback(request, **kwargs)
            if callback_return is not None:
                code, response = callback_return
                self._send_response(request, code, response)

            try:
                request_metrics.stop(self.clock, request, servlet_classname)
            except:
                pass

            return

        # Huh. No one wanted to handle that? Fiiiiiine. Send 400.
        raise UnrecognizedRequestError()

    def _send_response(self, request, code, response_json_object,
                       response_code_message=None):
        # could alternatively use request.notifyFinish() and flip a flag when
        # the Deferred fires, but since the flag is RIGHT THERE it seems like
        # a waste.
        if request._disconnected:
            logger.warn(
                "Not sending response to request %s, already disconnected.",
                request)
            return

        outgoing_responses_counter.inc(request.method, str(code))

        # TODO: Only enable CORS for the requests that need it.
        respond_with_json(
            request, code, response_json_object,
            send_cors=True,
            response_code_message=response_code_message,
            pretty_print=_request_user_agent_is_curl(request),
            version_string=self.version_string,
            canonical_json=self.canonical_json,
        )

class RequestMetrics(object):
    def start(self, clock):
        self.start = clock.time_msec()
        self.start_context = LoggingContext.current_context()

    def stop(self, clock, request, servlet_classname):
        context = LoggingContext.current_context()

        tag = ""
        if context:
            tag = context.tag

            if context != self.start_context:
                logger.warn(
                    "Context have unexpectedly changed %r, %r",
                    context, self.start_context
                )
                return

        incoming_requests_counter.inc(request.method, servlet_classname, tag)

        response_timer.inc_by(
            clock.time_msec() - self.start, request.method,
            servlet_classname, tag
        )

        ru_utime, ru_stime = context.get_resource_usage()

        response_ru_utime.inc_by(
            ru_utime, request.method, servlet_classname, tag
        )
        response_ru_stime.inc_by(
            ru_stime, request.method, servlet_classname, tag
        )
        response_db_txn_count.inc_by(
            context.db_txn_count, request.method, servlet_classname, tag
        )
        response_db_txn_duration.inc_by(
            context.db_txn_duration, request.method, servlet_classname, tag
        )

class RootRedirect(resource.Resource):
    """Redirects the root '/' path to another path."""

    def __init__(self, path):
        resource.Resource.__init__(self)
        self.url = path

    def render_GET(self, request):
        return redirectTo(self.url, request)

    def getChild(self, name, request):
        if len(name) == 0:
            return self  # select ourselves as the child to render
        return resource.Resource.getChild(self, name, request)

def respond_with_json(request, code, json_object, send_cors=False,
                      response_code_message=None, pretty_print=False,
                      version_string="", canonical_json=True):
    if pretty_print:
        json_bytes = encode_pretty_printed_json(json_object) + "\n"
    else:
        if canonical_json or synapse.events.USE_FROZEN_DICTS:
            json_bytes = encode_canonical_json(json_object)
        else:
            # ujson doesn't like frozen_dicts.
            json_bytes = ujson.dumps(json_object, ensure_ascii=False)

    return respond_with_json_bytes(
        request, code, json_bytes,
        send_cors=send_cors,
        response_code_message=response_code_message,
        version_string=version_string
    )

def respond_with_json_bytes(request, code, json_bytes, send_cors=False,
                            version_string="", response_code_message=None):
    """Sends encoded JSON in response to the given request.

    Args:
        request (twisted.web.http.Request): The http request to respond to.
        code (int): The HTTP response code.
        json_bytes (bytes): The json bytes to use as the response body.
        send_cors (bool): Whether to send Cross-Origin Resource Sharing headers
            http://www.w3.org/TR/cors/
    Returns:
        twisted.web.server.NOT_DONE_YET"""

    request.setResponseCode(code, message=response_code_message)
    request.setHeader(b"Content-Type", b"application/json")
    request.setHeader(b"Server", version_string)
    request.setHeader(b"Content-Length", b"%d" % (len(json_bytes),))

    if send_cors:
        request.setHeader("Access-Control-Allow-Origin", "*")
        request.setHeader("Access-Control-Allow-Methods",
                          "GET, POST, PUT, DELETE, OPTIONS")
        request.setHeader("Access-Control-Allow-Headers",
                          "Origin, X-Requested-With, Content-Type, Accept")

    request.write(json_bytes)
    finish_request(request)
    return NOT_DONE_YET

def finish_request(request):
    """ Finish writing the response to the request.

    Twisted throws a RuntimeException if the connection closed before the
    response was written but doesn't provide a convenient or reliable way to
    determine if the connection was closed. So we catch and log the RuntimeException

    You might think that ``request.notifyFinish`` could be used to tell if the
    request was finished. However the deferred it returns won't fire if the
    connection was already closed, meaning we'd have to have called the method
    right at the start of the request. By the time we want to write the response
    it will already be too late.
    """
    try:
        request.finish()
    except RuntimeError as e:
        logger.info("Connection disconnected before response was written: %r", e)

def _request_user_agent_is_curl(request):
    user_agents = request.requestHeaders.getRawHeaders(
        "User-Agent", default=[]
    )
    for user_agent in user_agents:
        if "curl" in user_agent:
            return True
    return False
<|endoftext|>


<commit_before>import os

import synapse.common as s_common

from OpenSSL import crypto

defdir = os.getenv('SYN_CERT_DIR')
if defdir is None:
    defdir = '~/.syn/certs'

def iterFqdnUp(fqdn):
    levs = fqdn.split('.')
    for i in range(len(levs)):
        yield '.'.join(levs[i:])

class CertDir:
    CRYPTO_NUMBITS = 4096

    def __init__(self, path=None):
        self.crypto_numbits = 4096

        if path is None:
            path = defdir

        s_common.gendir(path, 'cas')
        s_common.gendir(path, 'hosts')
        s_common.gendir(path, 'users')

        self.certdir = s_common.reqdir(path)

    def getPathJoin(self, *paths):
        return s_common.genpath(self.certdir, *paths)

    def getCaCert(self, name):
        return self._loadCertPath(self.getCaCertPath(name))

    def getHostCert(self, name):
        return self._loadCertPath(self.getHostCertPath(name))

    def getUserCert(self, name):
        return self._loadCertPath(self.getUserCertPath(name))

    def getClientCert(self, name):
        '''
        Loads the PKCS12 object for a given client certificate.

        Example:
            mypkcs12 = cdir.getClientCert('mycert')

        Args:
            name (str): The name of the client certificate.

        Returns:
            OpenSSL.crypto.PKCS12: The certificate if exists.
        '''
        return self._loadP12Path(self.getClientCertPath(name))

    def getCaKey(self, name):
        return self._loadKeyPath(self.getCaKeyPath(name))

    def getHostKey(self, name):
        return self._loadKeyPath(self.getHostKeyPath(name))

    def getUserKey(self, name):
        return self._loadKeyPath(self.getUserKeyPath(name))

    def _loadKeyPath(self, path):
        if path is None:
            return None

        byts = s_common.getbytes(path)
        if byts is None:
            return None

        return crypto.load_privatekey(crypto.FILETYPE_PEM, byts)

    def _loadCertPath(self, path):
        if path is None:
            return None

        byts = s_common.getbytes(path)
        if byts is None:
            return None

        return crypto.load_certificate(crypto.FILETYPE_PEM, byts)

    def _loadP12Path(self, path):
        if path is None:
            return None

        byts = s_common.getbytes(path)
        if byts is None:
            return None

        return crypto.load_pkcs12(byts)

    #def saveCaCert(self, cert):
    #def saveUserCert(self, cert):
    #def saveHostCert(self, cert):
    #def saveX509Cert(self, cert):
    #def loadX509Cert(self, path):

    def _genBasePkeyCert(self, name, pkey=None):

        if pkey is None:
            pkey = crypto.PKey()
            pkey.generate_key(crypto.TYPE_RSA, self.crypto_numbits)

        cert = crypto.X509()
        cert.set_pubkey(pkey)
        cert.gmtime_adj_notBefore(0)
        cert.gmtime_adj_notAfter(10 * 365 * 24 * 60 * 60)

        cert.set_serial_number(int(s_common.guid(), 16))
        cert.get_subject().CN = name

        return pkey, cert

    def _saveCertTo(self, cert, *paths):
        path = self.getPathJoin(*paths)
        if os.path.isfile(path):
            raise s_common.DupFileName(path=path)

        with s_common.genfile(path) as fd:
            fd.write(crypto.dump_certificate(crypto.FILETYPE_PEM, cert))

        return path

    def _saveP12To(self, cert, *paths):
        path = self.getPathJoin(*paths)
        if os.path.isfile(path):
            raise s_common.DupFileName(path=path)

        with s_common.genfile(path) as fd:
            fd.write(cert.export())

        return path

    def _savePkeyTo(self, pkey, *paths):
        path = self.getPathJoin(*paths)
        if os.path.isfile(path):
            raise s_common.DupFileName(path=path)

        with s_common.genfile(path) as fd:
            fd.write(crypto.dump_privatekey(crypto.FILETYPE_PEM, pkey))
        return path

    def genCaCert(self, name, signas=None, outp=None):
        pkey, cert = self._genBasePkeyCert(name)
        ext0 = crypto.X509Extension(b'basicConstraints', False, b'CA:TRUE')
        cert.add_extensions([ext0])

        if signas is not None:
            self.signCertAs(cert, signas)
        else:
            self.selfSignCert(cert, pkey)

        keypath = self._savePkeyTo(pkey, 'cas', '%s.key' % name)
        if outp is not None:
            outp.printf('key saved: %s' % (keypath,))

        crtpath = self._saveCertTo(cert, 'cas', '%s.crt' % name)
        if outp is not None:
            outp.printf('cert saved: %s' % (crtpath,))

        return pkey, cert

    def genHostCert(self, name, signas=None, outp=None, pkey=None, sans=None):
        pkey, cert = self._genBasePkeyCert(name, pkey=pkey)

        ext_sans = {'DNS:' + name}
        if isinstance(sans, str):
            ext_sans = ext_sans.union(sans.split(','))
        ext_sans = ','.join(sorted(ext_sans))

        cert.add_extensions([
            crypto.X509Extension(b'nsCertType', False, b'server'),
            crypto.X509Extension(b'keyUsage', False, b'digitalSignature,keyEncipherment'),
            crypto.X509Extension(b'extendedKeyUsage', False, b'serverAuth'),
            crypto.X509Extension(b'basicConstraints', False, b'CA:FALSE'),
            crypto.X509Extension(b'subjectAltName', False, ext_sans.encode('utf-8')),
        ])

        if signas is not None:
            self.signCertAs(cert, signas)
        else:
            self.selfSignCert(cert, pkey)

        if not pkey._only_public:
            keypath = self._savePkeyTo(pkey, 'hosts', '%s.key' % name)
            if outp is not None:
                outp.printf('key saved: %s' % (keypath,))

        crtpath = self._saveCertTo(cert, 'hosts', '%s.crt' % name)
        if outp is not None:
            outp.printf('cert saved: %s' % (crtpath,))

        return pkey, cert

    def genUserCert(self, name, signas=None, outp=None, pkey=None):

        pkey, cert = self._genBasePkeyCert(name, pkey=pkey)

        cert.add_extensions([
            crypto.X509Extension(b'nsCertType', False, b'client'),
            crypto.X509Extension(b'keyUsage', False, b'digitalSignature'),
            crypto.X509Extension(b'extendedKeyUsage', False, b'clientAuth'),
            crypto.X509Extension(b'basicConstraints', False, b'CA:FALSE'),
        ])

        if signas is not None:
            self.signCertAs(cert, signas)
        else:
            self.selfSignCert(cert, pkey)

        if not pkey._only_public:
            keypath = self._savePkeyTo(pkey, 'users', '%s.key' % name)
            if outp is not None:
                outp.printf('key saved: %s' % (keypath,))

        crtpath = self._saveCertTo(cert, 'users', '%s.crt' % name)
        if outp is not None:
            outp.printf('cert saved: %s' % (crtpath,))

        ccert = crypto.PKCS12()
        ccert.set_friendlyname(name.encode('utf-8'))
        ccert.set_certificate(cert)
        ccert.set_privatekey(pkey)

        if signas:
            cacert = self.getCaCert(signas)
            ccert.set_ca_certificates([cacert])

        crtpath = self._saveP12To(ccert, 'users', '%s.p12' % name)
        if outp is not None:
            outp.printf('client cert saved: %s' % (crtpath,))

        return pkey, cert

    def _loadCsrPath(self, path):
        byts = s_common.getbytes(path)
        if byts is None:
            return None
        return crypto.load_certificate_request(crypto.FILETYPE_PEM, byts)

    def genUserCsr(self, name, outp=None):
        return self._genPkeyCsr(name, 'users', outp=outp)

    def genHostCsr(self, name, outp=None):
        return self._genPkeyCsr(name, 'hosts', outp=outp)

    def signUserCsr(self, xcsr, signas, outp=None):
        pkey = xcsr.get_pubkey()
        name = xcsr.get_subject().CN
        return self.genUserCert(name, pkey=pkey, signas=signas, outp=outp)

    def signHostCsr(self, xcsr, signas, outp=None, sans=None):
        pkey = xcsr.get_pubkey()
        name = xcsr.get_subject().CN
        return self.genHostCert(name, pkey=pkey, signas=signas, outp=outp, sans=sans)

    def _genPkeyCsr(self, name, mode, outp=None):
        pkey = crypto.PKey()
        pkey.generate_key(crypto.TYPE_RSA, self.crypto_numbits)

        xcsr = crypto.X509Req()
        xcsr.get_subject().CN = name

        xcsr.set_pubkey(pkey)
        xcsr.sign(pkey, 'sha256')

        keypath = self._savePkeyTo(pkey, mode, '%s.key' % name)
        if outp is not None:
            outp.printf('key saved: %s' % (keypath,))

        csrpath = self.getPathJoin(mode, '%s.csr' % name)
        if os.path.isfile(csrpath):
            raise s_common.DupFileName(path=csrpath)

        with s_common.genfile(csrpath) as fd:
            fd.write(crypto.dump_certificate_request(crypto.FILETYPE_PEM, xcsr))

        if outp is not None:
            outp.printf('csr saved: %s' % (csrpath,))

    def signCertAs(self, cert, signas):
        cakey = self.getCaKey(signas)
        cacert = self.getCaCert(signas)

        cert.set_issuer(cacert.get_subject())
        cert.sign(cakey, 'sha256')

    def selfSignCert(self, cert, pkey):
        cert.set_issuer(cert.get_subject())
        cert.sign(pkey, 'sha256')

    def getUserForHost(self, user, host):
        for name in iterFqdnUp(host):
            usercert = '%s@%s' % (user, name)
            if self.isUserCert(usercert):
                return usercert

    def getCaCertPath(self, name):
        path = s_common.genpath(self.certdir, 'cas', '%s.crt' % name)
        if not os.path.isfile(path):
            return None
        return path

    def getCaKeyPath(self, name):
        path = s_common.genpath(self.certdir, 'cas', '%s.key' % name)
        if not os.path.isfile(path):
            return None
        return path

    def getHostCertPath(self, name):
        path = s_common.genpath(self.certdir, 'hosts', '%s.crt' % name)
        if not os.path.isfile(path):
            return None
        return path

    def getHostKeyPath(self, name):
        path = s_common.genpath(self.certdir, 'hosts', '%s.key' % name)
        if not os.path.isfile(path):
            return None
        return path

    def getUserCertPath(self, name):
        path = s_common.genpath(self.certdir, 'users', '%s.crt' % name)
        if not os.path.isfile(path):
            return None
        return path

    def getClientCertPath(self, name):
        '''
        Gets the path to a client certificate.

        Example:
            mypath = cdir.getClientCertPath('mycert')

        Args:
            name (str): The name of the client certificate.

        Returns:
            str: The path if exists.
        '''
        path = s_common.genpath(self.certdir, 'users', '%s.p12' % name)
        if not os.path.isfile(path):
            return None
        return path

    def getUserKeyPath(self, name):
        path = s_common.genpath(self.certdir, 'users', '%s.key' % name)
        if not os.path.isfile(path):
            return None
        return path

    def getUserCaPath(self, name):
        cert = self.getUserCert(name)
        if cert is None:
            return None

        subj = cert.get_issuer()

        capath = self.getPathJoin('cas', '%s.crt' % subj.CN)
        if not os.path.isfile(capath):
            return None

        return capath

    def getHostCaPath(self, name):
        cert = self.getHostCert(name)
        if cert is None:
            return None

        subj = cert.get_issuer()

        capath = self.getPathJoin('cas', '%s.crt' % subj.CN)
        if not os.path.isfile(capath):
            return None

        return capath

    def isUserCert(self, name):
        crtpath = self.getPathJoin('users', '%s.crt' % name)
        return os.path.isfile(crtpath)

    def isCaCert(self, name):
        crtpath = self.getPathJoin('cas', '%s.crt' % name)
        return os.path.isfile(crtpath)

    def isHostCert(self, name):
        crtpath = self.getPathJoin('hosts', '%s.crt' % name)
        return os.path.isfile(crtpath)

    def isClientCert(self, name):
        '''
        Checks if a client certificate exists.

        Example:
            exists = cdir.isClientCert('mycert')

        Args:
            name (str): The name of the client certificate.

        Returns:
            bool: True if the certificate is present, False otherwise.
        '''
        crtpath = self.getPathJoin('users', '%s.p12' % name)
        return os.path.isfile(crtpath)
<commit_msg>Remove class variable<commit_after>import os

import synapse.common as s_common

from OpenSSL import crypto

defdir = os.getenv('SYN_CERT_DIR')
if defdir is None:
    defdir = '~/.syn/certs'

def iterFqdnUp(fqdn):
    levs = fqdn.split('.')
    for i in range(len(levs)):
        yield '.'.join(levs[i:])

class CertDir:

    def __init__(self, path=None):
        self.crypto_numbits = 4096

        if path is None:
            path = defdir

        s_common.gendir(path, 'cas')
        s_common.gendir(path, 'hosts')
        s_common.gendir(path, 'users')

        self.certdir = s_common.reqdir(path)

    def getPathJoin(self, *paths):
        return s_common.genpath(self.certdir, *paths)

    def getCaCert(self, name):
        return self._loadCertPath(self.getCaCertPath(name))

    def getHostCert(self, name):
        return self._loadCertPath(self.getHostCertPath(name))

    def getUserCert(self, name):
        return self._loadCertPath(self.getUserCertPath(name))

    def getClientCert(self, name):
        '''
        Loads the PKCS12 object for a given client certificate.

        Example:
            mypkcs12 = cdir.getClientCert('mycert')

        Args:
            name (str): The name of the client certificate.

        Returns:
            OpenSSL.crypto.PKCS12: The certificate if exists.
        '''
        return self._loadP12Path(self.getClientCertPath(name))

    def getCaKey(self, name):
        return self._loadKeyPath(self.getCaKeyPath(name))

    def getHostKey(self, name):
        return self._loadKeyPath(self.getHostKeyPath(name))

    def getUserKey(self, name):
        return self._loadKeyPath(self.getUserKeyPath(name))

    def _loadKeyPath(self, path):
        if path is None:
            return None

        byts = s_common.getbytes(path)
        if byts is None:
            return None

        return crypto.load_privatekey(crypto.FILETYPE_PEM, byts)

    def _loadCertPath(self, path):
        if path is None:
            return None

        byts = s_common.getbytes(path)
        if byts is None:
            return None

        return crypto.load_certificate(crypto.FILETYPE_PEM, byts)

    def _loadP12Path(self, path):
        if path is None:
            return None

        byts = s_common.getbytes(path)
        if byts is None:
            return None

        return crypto.load_pkcs12(byts)

    #def saveCaCert(self, cert):
    #def saveUserCert(self, cert):
    #def saveHostCert(self, cert):
    #def saveX509Cert(self, cert):
    #def loadX509Cert(self, path):

    def _genBasePkeyCert(self, name, pkey=None):

        if pkey is None:
            pkey = crypto.PKey()
            pkey.generate_key(crypto.TYPE_RSA, self.crypto_numbits)

        cert = crypto.X509()
        cert.set_pubkey(pkey)
        cert.gmtime_adj_notBefore(0)
        cert.gmtime_adj_notAfter(10 * 365 * 24 * 60 * 60)

        cert.set_serial_number(int(s_common.guid(), 16))
        cert.get_subject().CN = name

        return pkey, cert

    def _saveCertTo(self, cert, *paths):
        path = self.getPathJoin(*paths)
        if os.path.isfile(path):
            raise s_common.DupFileName(path=path)

        with s_common.genfile(path) as fd:
            fd.write(crypto.dump_certificate(crypto.FILETYPE_PEM, cert))

        return path

    def _saveP12To(self, cert, *paths):
        path = self.getPathJoin(*paths)
        if os.path.isfile(path):
            raise s_common.DupFileName(path=path)

        with s_common.genfile(path) as fd:
            fd.write(cert.export())

        return path

    def _savePkeyTo(self, pkey, *paths):
        path = self.getPathJoin(*paths)
        if os.path.isfile(path):
            raise s_common.DupFileName(path=path)

        with s_common.genfile(path) as fd:
            fd.write(crypto.dump_privatekey(crypto.FILETYPE_PEM, pkey))
        return path

    def genCaCert(self, name, signas=None, outp=None):
        pkey, cert = self._genBasePkeyCert(name)
        ext0 = crypto.X509Extension(b'basicConstraints', False, b'CA:TRUE')
        cert.add_extensions([ext0])

        if signas is not None:
            self.signCertAs(cert, signas)
        else:
            self.selfSignCert(cert, pkey)

        keypath = self._savePkeyTo(pkey, 'cas', '%s.key' % name)
        if outp is not None:
            outp.printf('key saved: %s' % (keypath,))

        crtpath = self._saveCertTo(cert, 'cas', '%s.crt' % name)
        if outp is not None:
            outp.printf('cert saved: %s' % (crtpath,))

        return pkey, cert

    def genHostCert(self, name, signas=None, outp=None, pkey=None, sans=None):
        pkey, cert = self._genBasePkeyCert(name, pkey=pkey)

        ext_sans = {'DNS:' + name}
        if isinstance(sans, str):
            ext_sans = ext_sans.union(sans.split(','))
        ext_sans = ','.join(sorted(ext_sans))

        cert.add_extensions([
            crypto.X509Extension(b'nsCertType', False, b'server'),
            crypto.X509Extension(b'keyUsage', False, b'digitalSignature,keyEncipherment'),
            crypto.X509Extension(b'extendedKeyUsage', False, b'serverAuth'),
            crypto.X509Extension(b'basicConstraints', False, b'CA:FALSE'),
            crypto.X509Extension(b'subjectAltName', False, ext_sans.encode('utf-8')),
        ])

        if signas is not None:
            self.signCertAs(cert, signas)
        else:
            self.selfSignCert(cert, pkey)

        if not pkey._only_public:
            keypath = self._savePkeyTo(pkey, 'hosts', '%s.key' % name)
            if outp is not None:
                outp.printf('key saved: %s' % (keypath,))

        crtpath = self._saveCertTo(cert, 'hosts', '%s.crt' % name)
        if outp is not None:
            outp.printf('cert saved: %s' % (crtpath,))

        return pkey, cert

    def genUserCert(self, name, signas=None, outp=None, pkey=None):

        pkey, cert = self._genBasePkeyCert(name, pkey=pkey)

        cert.add_extensions([
            crypto.X509Extension(b'nsCertType', False, b'client'),
            crypto.X509Extension(b'keyUsage', False, b'digitalSignature'),
            crypto.X509Extension(b'extendedKeyUsage', False, b'clientAuth'),
            crypto.X509Extension(b'basicConstraints', False, b'CA:FALSE'),
        ])

        if signas is not None:
            self.signCertAs(cert, signas)
        else:
            self.selfSignCert(cert, pkey)

        if not pkey._only_public:
            keypath = self._savePkeyTo(pkey, 'users', '%s.key' % name)
            if outp is not None:
                outp.printf('key saved: %s' % (keypath,))

        crtpath = self._saveCertTo(cert, 'users', '%s.crt' % name)
        if outp is not None:
            outp.printf('cert saved: %s' % (crtpath,))

        ccert = crypto.PKCS12()
        ccert.set_friendlyname(name.encode('utf-8'))
        ccert.set_certificate(cert)
        ccert.set_privatekey(pkey)

        if signas:
            cacert = self.getCaCert(signas)
            ccert.set_ca_certificates([cacert])

        crtpath = self._saveP12To(ccert, 'users', '%s.p12' % name)
        if outp is not None:
            outp.printf('client cert saved: %s' % (crtpath,))

        return pkey, cert

    def _loadCsrPath(self, path):
        byts = s_common.getbytes(path)
        if byts is None:
            return None
        return crypto.load_certificate_request(crypto.FILETYPE_PEM, byts)

    def genUserCsr(self, name, outp=None):
        return self._genPkeyCsr(name, 'users', outp=outp)

    def genHostCsr(self, name, outp=None):
        return self._genPkeyCsr(name, 'hosts', outp=outp)

    def signUserCsr(self, xcsr, signas, outp=None):
        pkey = xcsr.get_pubkey()
        name = xcsr.get_subject().CN
        return self.genUserCert(name, pkey=pkey, signas=signas, outp=outp)

    def signHostCsr(self, xcsr, signas, outp=None, sans=None):
        pkey = xcsr.get_pubkey()
        name = xcsr.get_subject().CN
        return self.genHostCert(name, pkey=pkey, signas=signas, outp=outp, sans=sans)

    def _genPkeyCsr(self, name, mode, outp=None):
        pkey = crypto.PKey()
        pkey.generate_key(crypto.TYPE_RSA, self.crypto_numbits)

        xcsr = crypto.X509Req()
        xcsr.get_subject().CN = name

        xcsr.set_pubkey(pkey)
        xcsr.sign(pkey, 'sha256')

        keypath = self._savePkeyTo(pkey, mode, '%s.key' % name)
        if outp is not None:
            outp.printf('key saved: %s' % (keypath,))

        csrpath = self.getPathJoin(mode, '%s.csr' % name)
        if os.path.isfile(csrpath):
            raise s_common.DupFileName(path=csrpath)

        with s_common.genfile(csrpath) as fd:
            fd.write(crypto.dump_certificate_request(crypto.FILETYPE_PEM, xcsr))

        if outp is not None:
            outp.printf('csr saved: %s' % (csrpath,))

    def signCertAs(self, cert, signas):
        cakey = self.getCaKey(signas)
        cacert = self.getCaCert(signas)

        cert.set_issuer(cacert.get_subject())
        cert.sign(cakey, 'sha256')

    def selfSignCert(self, cert, pkey):
        cert.set_issuer(cert.get_subject())
        cert.sign(pkey, 'sha256')

    def getUserForHost(self, user, host):
        for name in iterFqdnUp(host):
            usercert = '%s@%s' % (user, name)
            if self.isUserCert(usercert):
                return usercert

    def getCaCertPath(self, name):
        path = s_common.genpath(self.certdir, 'cas', '%s.crt' % name)
        if not os.path.isfile(path):
            return None
        return path

    def getCaKeyPath(self, name):
        path = s_common.genpath(self.certdir, 'cas', '%s.key' % name)
        if not os.path.isfile(path):
            return None
        return path

    def getHostCertPath(self, name):
        path = s_common.genpath(self.certdir, 'hosts', '%s.crt' % name)
        if not os.path.isfile(path):
            return None
        return path

    def getHostKeyPath(self, name):
        path = s_common.genpath(self.certdir, 'hosts', '%s.key' % name)
        if not os.path.isfile(path):
            return None
        return path

    def getUserCertPath(self, name):
        path = s_common.genpath(self.certdir, 'users', '%s.crt' % name)
        if not os.path.isfile(path):
            return None
        return path

    def getClientCertPath(self, name):
        '''
        Gets the path to a client certificate.

        Example:
            mypath = cdir.getClientCertPath('mycert')

        Args:
            name (str): The name of the client certificate.

        Returns:
            str: The path if exists.
        '''
        path = s_common.genpath(self.certdir, 'users', '%s.p12' % name)
        if not os.path.isfile(path):
            return None
        return path

    def getUserKeyPath(self, name):
        path = s_common.genpath(self.certdir, 'users', '%s.key' % name)
        if not os.path.isfile(path):
            return None
        return path

    def getUserCaPath(self, name):
        cert = self.getUserCert(name)
        if cert is None:
            return None

        subj = cert.get_issuer()

        capath = self.getPathJoin('cas', '%s.crt' % subj.CN)
        if not os.path.isfile(capath):
            return None

        return capath

    def getHostCaPath(self, name):
        cert = self.getHostCert(name)
        if cert is None:
            return None

        subj = cert.get_issuer()

        capath = self.getPathJoin('cas', '%s.crt' % subj.CN)
        if not os.path.isfile(capath):
            return None

        return capath

    def isUserCert(self, name):
        crtpath = self.getPathJoin('users', '%s.crt' % name)
        return os.path.isfile(crtpath)

    def isCaCert(self, name):
        crtpath = self.getPathJoin('cas', '%s.crt' % name)
        return os.path.isfile(crtpath)

    def isHostCert(self, name):
        crtpath = self.getPathJoin('hosts', '%s.crt' % name)
        return os.path.isfile(crtpath)

    def isClientCert(self, name):
        '''
        Checks if a client certificate exists.

        Example:
            exists = cdir.isClientCert('mycert')

        Args:
            name (str): The name of the client certificate.

        Returns:
            bool: True if the certificate is present, False otherwise.
        '''
        crtpath = self.getPathJoin('users', '%s.p12' % name)
        return os.path.isfile(crtpath)
<|endoftext|>


<commit_before>import os
import asyncio
import logging
import contextlib
import collections
import contextvars
import dataclasses

from typing import Optional

import synapse.exc as s_exc
import synapse.common as s_common

import synapse.lib.cache as s_cache
import synapse.lib.msgpack as s_msgpack

logger = logging.getLogger(__name__)

Conditions = set((
    'tag:add',
    'tag:del',
    'node:add',
    'node:del',
    'prop:set',
))

RecursionDepth = contextvars.ContextVar('RecursionDepth', default=0)

class Triggers:

    @dataclasses.dataclass
    class Rule:
        ver: int  # version: must be 0
        cond: str  # condition from above list
        user: str  # username
        storm: str  # story query
        form: Optional[str] = dataclasses.field(default=None) # form name
        tag: Optional[str] = dataclasses.field(default=None) # tag name
        prop: Optional[str] = dataclasses.field(default=None) # property name

        def __post_init__(self):
            if self.ver != 0:
                raise s_exc.BadOptValu(mesg='Unexpected rule version')
            if self.cond not in Conditions:
                raise s_exc.BadOptValu(mesg='Invalid trigger condition')
            if self.cond in ('node:add', 'node:del') and self.form is None:
                raise s_exc.BadOptValu(mesg='form must be present for node:add or node:del')
            if self.cond in ('node:add', 'node:del') and self.tag is not None:
                raise s_exc.BadOptValu(mesg='tag must not be present for node:add or node:del')
            if self.cond == 'prop:set' and (self.form is not None or self.tag is not None):
                raise s_exc.BadOptValu(mesg='form and tag must not be present for prop:set')
            if self.cond in ('tag:add', 'tag:del') and self.tag is None:
                raise s_exc.BadOptValu(mesg='missing tag')
            if self.prop is not None and self.cond != 'prop:set':
                raise s_exc.BadOptValu(mesg='prop parameter invalid')
            if self.cond == 'prop:set' and self.prop is None:
                raise s_exc.BadOptValu(mesg='missing prop parameter')

        def en(self):
            return s_msgpack.en(dataclasses.asdict(self))

        async def execute(self, node, vars=None):
            '''
            Actually execute the query
            '''
            opts = {}
            if vars is not None:
                opts['vars'] = vars

            if node.snap.core.auth is not None:
                user = node.snap.core.auth.users.get(self.user)
                if user is None:
                    logger.warning('Unknown user %s in stored trigger', self.user)
                    return
            else:
                user = None

            try:
                await s_common.aspin(node.storm(self.storm, opts=opts, user=user))
            except asyncio.CancelledError: # pragma: no cover
                raise
            except Exception:
                logger.exception('Trigger encountered exception running storm query %s', self.storm)

    def __init__(self, core):
        '''
        Initialize a cortex triggers subsystem.

        Note:
            Triggers will not fire until enable() is called.
        '''
        self.core = core
        self._rules = {}

        self.trigdb = self.core.slab.initdb('triggers')

        self.tagadd = collections.defaultdict(list)    # (form, tag): rule
        self.tagdel = collections.defaultdict(list)    # (form, tag): rule

        self.tagaddglobs = collections.defaultdict(s_cache.TagGlobs)    # form: TagGlobs
        self.tagdelglobs = collections.defaultdict(s_cache.TagGlobs)    # form: TagGlobs

        self.nodeadd = collections.defaultdict(list)   # form: rule
        self.nodedel = collections.defaultdict(list)   # form: rule
        self.propset = collections.defaultdict(list)   # prop: rule

        self._load_all(self.core.slab)

    async def runNodeAdd(self, node):
        with self._recursion_check():
            [await rule.execute(node) for rule in self.nodeadd.get(node.form.name, ())]

    async def runNodeDel(self, node):
        with self._recursion_check():
            [await rule.execute(node) for rule in self.nodedel.get(node.form.name, ())]

    async def runPropSet(self, node, prop, oldv):
        with self._recursion_check():
            [await rule.execute(node) for rule in self.propset.get(prop.full, ())]

    async def runTagAdd(self, node, tag):

        vars = {'tag': tag}
        with self._recursion_check():

            for rule in self.tagadd.get((node.form.name, tag), ()):
                await rule.execute(node, vars=vars)

            for rule in self.tagadd.get((None, tag), ()):
                await rule.execute(node, vars=vars)

            # check for form specific globs
            globs = self.tagaddglobs.get(node.form.name)
            if globs is not None:
                for expr, rule in globs.get(tag):
                    await rule.execute(node, vars=vars)

            # check for form agnostic globs
            globs = self.tagaddglobs.get(None)
            if globs is not None:
                for expr, rule in globs.get(tag):
                    await rule.execute(node, vars=vars)

    async def runTagDel(self, node, tag):

        vars = {'tag': tag}
        with self._recursion_check():

            for rule in self.tagdel.get((node.form.name, tag), ()):
                await rule.execute(node, vars=vars)

            for rule in self.tagdel.get((None, tag), ()):
                await rule.execute(node, vars=vars)

            # check for form specific globs
            globs = self.tagdelglobs.get(node.form.name)
            if globs is not None:
                for expr, rule in globs.get(tag):
                    await rule.execute(node, vars=vars)

            # check for form agnostic globs
            globs = self.tagdelglobs.get(None)
            if globs is not None:
                for expr, rule in globs.get(tag):
                    await rule.execute(node, vars=vars)

    def _load_all(self, slab):
        for iden, val in self.core.slab.scanByRange(b'', db=self.trigdb):
            try:
                ruledict = s_msgpack.un(val)
                ver = ruledict.pop('ver')
                cond = ruledict.pop('cond')
                user = ruledict.pop('user')
                query = ruledict.pop('storm')
                self._load_rule(iden, ver, cond, user, query, info=ruledict)
            except Exception as e:
                logger.warning('Invalid rule %r found in storage: %r', iden, e)
                continue

    def _load_rule(self, iden, ver, cond, user, query, info):

        rule = Triggers.Rule(ver, cond, user, query, **info)

        # Make sure the query parses
        self.core.getStormQuery(rule.storm)

        self._rules[iden] = rule

        if rule.cond == 'node:add':
            self.nodeadd[rule.form].append(rule)
            return rule

        if rule.cond == 'node:del':
            self.nodedel[rule.form].append(rule)
            return rule

        if rule.cond == 'prop:set':
            self.propset[rule.prop].append(rule)
            return rule

        if rule.cond == 'tag:add':

            if rule.tag.find('*') == -1:
                self.tagadd[(rule.form, rule.tag)].append(rule)
                return rule

            # we have a glob add
            self.tagaddglobs[rule.form].add(rule.tag, rule)
            return rule

        if rule.cond == 'tag:del':

            if rule.tag.find('*') == -1:
                self.tagdel[(rule.form, rule.tag)].append(rule)
                return rule

            self.tagdelglobs[rule.form].add(rule.tag, rule)
            return rule

        raise s_exc.NoSuchCond(name=rule.cond)

    def list(self):
        return [(iden, dataclasses.asdict(rule)) for iden, rule in self._rules.items()]

    def mod(self, iden, query):
        rule = self._rules.get(iden)
        if rule is None:
            raise s_exc.NoSuchIden()

        self.core.getStormQuery(query)

        rule.storm = query
        self.core.slab.put(iden, rule.en(), db=self.trigdb)

    @contextlib.contextmanager
    def _recursion_check(self):

        depth = RecursionDepth.get()
        if depth > 64:
            raise s_exc.RecursionLimitHit(mesg='Hit trigger limit')

        token = RecursionDepth.set(depth + 1)

        try:
            yield

        finally:
            RecursionDepth.reset(token)

    def add(self, username, condition, query, info):

        iden = os.urandom(16)

        if not query:
            raise ValueError('empty query')

        # Check the storm query if we can
        self.core.getStormQuery(query)

        rule = self._load_rule(iden, 0, condition, username, query, info=info)
        self.core.slab.put(iden, rule.en(), db=self.trigdb)
        return iden

    def delete(self, iden, persistent=True):
        '''
        Args:
            persistent (bool): if True, removes from persistent storage as well
        '''
        rule = self._rules.pop(iden, None)
        if rule is None:
            raise s_exc.NoSuchIden()

        if rule.cond == 'node:add':
            self.nodeadd[rule.form].remove(rule)
            return

        if rule.cond == 'node:del':
            self.nodedel[rule.form].remove(rule)
            return

        if rule.cond == 'prop:set':
            self.propset[rule.prop].remove(rule)
            return

        if rule.cond == 'tag:add':

            if rule.tag.find('*') == -1:
                self.tagadd[(rule.form, rule.tag)].remove(rule)
                return

            globs = self.tagaddglobs.get(rule.form)
            globs.rem(rule.tag, rule)
            return

        if rule.cond == 'tag:del':

            if rule.tag.find('*') == -1:
                self.tagdel[(rule.form, rule.tag)].remove(rule)
                return

            globs = self.tagdelglobs.get(rule.form)
            globs.rem(rule.tag, rule)
            return

    def get(self, iden):
        rule = self._rules.get(iden)
        if rule is None:
            raise s_exc.NoSuchIden()
        return dataclasses.asdict(rule)
<commit_msg>fixes from review<commit_after>import os
import asyncio
import logging
import contextlib
import collections
import contextvars
import dataclasses

from typing import Optional

import synapse.exc as s_exc
import synapse.common as s_common

import synapse.lib.cache as s_cache
import synapse.lib.msgpack as s_msgpack

logger = logging.getLogger(__name__)

Conditions = set((
    'tag:add',
    'tag:del',
    'node:add',
    'node:del',
    'prop:set',
))

RecursionDepth = contextvars.ContextVar('RecursionDepth', default=0)

class Triggers:

    @dataclasses.dataclass
    class Rule:
        ver: int  # version: must be 0
        cond: str  # condition from above list
        user: str  # username
        storm: str  # story query
        form: Optional[str] = dataclasses.field(default=None) # form name
        tag: Optional[str] = dataclasses.field(default=None) # tag name
        prop: Optional[str] = dataclasses.field(default=None) # property name

        def __post_init__(self):
            if self.ver != 0:
                raise s_exc.BadOptValu(mesg='Unexpected rule version')
            if self.cond not in Conditions:
                raise s_exc.BadOptValu(mesg='Invalid trigger condition')
            if self.cond in ('node:add', 'node:del') and self.form is None:
                raise s_exc.BadOptValu(mesg='form must be present for node:add or node:del')
            if self.cond in ('node:add', 'node:del') and self.tag is not None:
                raise s_exc.BadOptValu(mesg='tag must not be present for node:add or node:del')
            if self.cond == 'prop:set' and (self.form is not None or self.tag is not None):
                raise s_exc.BadOptValu(mesg='form and tag must not be present for prop:set')
            if self.cond in ('tag:add', 'tag:del') and self.tag is None:
                raise s_exc.BadOptValu(mesg='missing tag')
            if self.prop is not None and self.cond != 'prop:set':
                raise s_exc.BadOptValu(mesg='prop parameter invalid')
            if self.cond == 'prop:set' and self.prop is None:
                raise s_exc.BadOptValu(mesg='missing prop parameter')

        def en(self):
            return s_msgpack.en(dataclasses.asdict(self))

        async def execute(self, node, vars=None):
            '''
            Actually execute the query
            '''
            opts = {}
            if vars is not None:
                opts['vars'] = vars

            if node.snap.core.auth is not None:
                user = node.snap.core.auth.users.get(self.user)
                if user is None:
                    logger.warning('Unknown user %s in stored trigger', self.user)
                    return
            else:
                user = None

            try:
                await s_common.aspin(node.storm(self.storm, opts=opts, user=user))
            except asyncio.CancelledError: # pragma: no cover
                raise
            except Exception:
                logger.exception('Trigger encountered exception running storm query %s', self.storm)

    def __init__(self, core):
        '''
        Initialize a cortex triggers subsystem.
        '''
        self.core = core
        self._rules = {}

        self.trigdb = self.core.slab.initdb('triggers')

        self.tagadd = collections.defaultdict(list)    # (form, tag): rule
        self.tagdel = collections.defaultdict(list)    # (form, tag): rule

        self.tagaddglobs = collections.defaultdict(s_cache.TagGlobs)    # form: TagGlobs
        self.tagdelglobs = collections.defaultdict(s_cache.TagGlobs)    # form: TagGlobs

        self.nodeadd = collections.defaultdict(list)   # form: rule
        self.nodedel = collections.defaultdict(list)   # form: rule
        self.propset = collections.defaultdict(list)   # prop: rule

        self._load_all(self.core.slab)

    async def runNodeAdd(self, node):
        with self._recursion_check():
            [await rule.execute(node) for rule in self.nodeadd.get(node.form.name, ())]

    async def runNodeDel(self, node):
        with self._recursion_check():
            [await rule.execute(node) for rule in self.nodedel.get(node.form.name, ())]

    async def runPropSet(self, node, prop, oldv):
        with self._recursion_check():
            [await rule.execute(node) for rule in self.propset.get(prop.full, ())]

    async def runTagAdd(self, node, tag):

        vars = {'tag': tag}
        with self._recursion_check():

            for rule in self.tagadd.get((node.form.name, tag), ()):
                await rule.execute(node, vars=vars)

            for rule in self.tagadd.get((None, tag), ()):
                await rule.execute(node, vars=vars)

            # check for form specific globs
            globs = self.tagaddglobs.get(node.form.name)
            if globs is not None:
                for expr, rule in globs.get(tag):
                    await rule.execute(node, vars=vars)

            # check for form agnostic globs
            globs = self.tagaddglobs.get(None)
            if globs is not None:
                for expr, rule in globs.get(tag):
                    await rule.execute(node, vars=vars)

    async def runTagDel(self, node, tag):

        vars = {'tag': tag}
        with self._recursion_check():

            for rule in self.tagdel.get((node.form.name, tag), ()):
                await rule.execute(node, vars=vars)

            for rule in self.tagdel.get((None, tag), ()):
                await rule.execute(node, vars=vars)

            # check for form specific globs
            globs = self.tagdelglobs.get(node.form.name)
            if globs is not None:
                for expr, rule in globs.get(tag):
                    await rule.execute(node, vars=vars)

            # check for form agnostic globs
            globs = self.tagdelglobs.get(None)
            if globs is not None:
                for expr, rule in globs.get(tag):
                    await rule.execute(node, vars=vars)

    def _load_all(self, slab):
        for iden, val in self.core.slab.scanByFull(db=self.trigdb):
            try:
                ruledict = s_msgpack.un(val)
                ver = ruledict.pop('ver')
                cond = ruledict.pop('cond')
                user = ruledict.pop('user')
                query = ruledict.pop('storm')
                self._load_rule(iden, ver, cond, user, query, info=ruledict)
            except Exception as e:
                logger.warning('Invalid rule %r found in storage: %r', iden, e)
                continue

    def _load_rule(self, iden, ver, cond, user, query, info):

        rule = Triggers.Rule(ver, cond, user, query, **info)

        # Make sure the query parses
        self.core.getStormQuery(rule.storm)

        self._rules[iden] = rule

        if rule.cond == 'node:add':
            self.nodeadd[rule.form].append(rule)
            return rule

        if rule.cond == 'node:del':
            self.nodedel[rule.form].append(rule)
            return rule

        if rule.cond == 'prop:set':
            self.propset[rule.prop].append(rule)
            return rule

        if rule.cond == 'tag:add':

            if '*' not in rule.tag:
                self.tagadd[(rule.form, rule.tag)].append(rule)
                return rule

            # we have a glob add
            self.tagaddglobs[rule.form].add(rule.tag, rule)
            return rule

        if rule.cond == 'tag:del':

            if '*' not in rule.tag:
                self.tagdel[(rule.form, rule.tag)].append(rule)
                return rule

            self.tagdelglobs[rule.form].add(rule.tag, rule)
            return rule

        raise s_exc.NoSuchCond(name=rule.cond)

    def list(self):
        return [(iden, dataclasses.asdict(rule)) for iden, rule in self._rules.items()]

    def mod(self, iden, query):
        rule = self._rules.get(iden)
        if rule is None:
            raise s_exc.NoSuchIden()

        self.core.getStormQuery(query)

        rule.storm = query
        self.core.slab.put(iden, rule.en(), db=self.trigdb)

    @contextlib.contextmanager
    def _recursion_check(self):

        depth = RecursionDepth.get()
        if depth > 64:
            raise s_exc.RecursionLimitHit(mesg='Hit trigger limit')

        token = RecursionDepth.set(depth + 1)

        try:
            yield

        finally:
            RecursionDepth.reset(token)

    def add(self, username, condition, query, info):

        iden = os.urandom(16)

        if not query:
            raise ValueError('empty query')

        self.core.getStormQuery(query)

        rule = self._load_rule(iden, 0, condition, username, query, info=info)
        self.core.slab.put(iden, rule.en(), db=self.trigdb)
        return iden

    def delete(self, iden):

        rule = self._rules.pop(iden, None)
        if rule is None:
            raise s_exc.NoSuchIden()

        self.core.slab.delete(iden, db=self.trigdb)

        if rule.cond == 'node:add':
            self.nodeadd[rule.form].remove(rule)
            return

        if rule.cond == 'node:del':
            self.nodedel[rule.form].remove(rule)
            return

        if rule.cond == 'prop:set':
            self.propset[rule.prop].remove(rule)
            return

        if rule.cond == 'tag:add':

            if '*' not in rule.tag:
                self.tagadd[(rule.form, rule.tag)].remove(rule)
                return

            globs = self.tagaddglobs.get(rule.form)
            globs.rem(rule.tag, rule)
            return

        if rule.cond == 'tag:del':

            if '*' not in rule.tag:
                self.tagdel[(rule.form, rule.tag)].remove(rule)
                return

            globs = self.tagdelglobs.get(rule.form)
            globs.rem(rule.tag, rule)
            return

    def get(self, iden):
        rule = self._rules.get(iden)
        if rule is None:
            raise s_exc.NoSuchIden()
        return dataclasses.asdict(rule)
<|endoftext|>


<commit_before>from __future__ import unicode_literals

from copy import deepcopy
import re
from base58 import b58decode
from mediachain.ingestion.asset_loader import load_asset, process_asset
from mediachain.datastore.dynamo import get_db
from mediachain.datastore.data_objects import MultihashReference
from grpc.framework.interfaces.face.face import NetworkError

class Writer(object):
    def __init__(self, transactor, datastore=None,
                 download_remote_assets=False):
        self.transactor = transactor
        self.datastore = datastore or get_db()
        self.download_remote_assets = download_remote_assets

    def write_dataset(self, dataset_iterator):
        translator_id = dataset_iterator.translator.translator_id()

        for result in dataset_iterator:
            translated = result['translated']
            raw = result['raw_content']
            local_assets = result.get('local_assets', {})
            refs = self.submit_translator_output(translator_id, translated,
                                                 raw, local_assets)
            print('Inserted canonical: {}'.format(refs['canonical']))

    def submit_translator_output(self,
                                 translator_id,
                                 translated,
                                 raw_content,
                                 local_assets):
        common_meta = {'translator': translator_id}
        meta_source = self.store_raw(raw_content)

        canonical = translated['canonical']
        canonical = self.flatten_record(canonical, common_meta,
                                        meta_source, local_assets)
        canonical_ref = self.submit_object(canonical)

        chain = translated.get('chain', [])
        chain_refs = []
        for cell in chain:
            cell = self.flatten_record(cell, common_meta,
                                       meta_source, local_assets)
            cell['ref'] = canonical_ref.to_map()
            chain_ref = self.submit_object(cell)
            chain_refs += chain_ref.multihash_base58()
        return {
            'canonical': canonical_ref.multihash_base58(),
            'chain': chain_refs
        }

    def flatten_record(self, record, common_meta, meta_source, local_assets):
        record = deepcopy(record)
        assets = {k: v for k, v in record.iteritems()
                  if is_asset(v)}

        for k, a in assets.iteritems():
            local = local_assets.get(k, None)
            ref = self.store_asset(k, local, a)
            if ref is None:
                print('Unable to store asset with key {}, removing'.format(k))
                del record[k]
            else:
                record[k] = ref.to_map()

        objects = {k: v for k, v in record.iteritems()
                   if isinstance(v, dict)}

        for k, o in objects.iteritems():
            flat = self.flatten_record(o, common_meta,
                                       meta_source, local_assets)
            if is_mediachain_object(o):
                ref = self.submit_object(flat)
                record[k] = ref.to_map()
            else:
                record[k] = flat

        if is_mediachain_object(record):
            del record['__mediachain_object__']
            record['metaSource'] = meta_source.to_map()
            record['meta'].update(common_meta)
        return record

    def store_raw(self, raw_content):
        ref = self.datastore.put(raw_content)
        return MultihashReference.from_base58(ref)

    def store_asset(self, name, local_asset, remote_asset):
        data = load_asset(local_asset)
        if data is None and self.download_remote_assets:
            data = load_asset(remote_asset)
        if data is None:
            return None
        data = process_asset(name, data)
        return MultihashReference.from_base58(self.datastore.put(data))

    def submit_object(self, obj):
        try:
            if is_canonical(obj):
                return self.transactor.insert_canonical(obj)
            else:
                return self.transactor.update_chain(obj)
        except NetworkError as e:
            ref_str = duplicate_canonical_ref(e.details)
            if ref_str is None:
                raise

        return MultihashReference.from_base58(ref_str)

def is_tagged_dict(tag, value):
    if not isinstance(value, dict):
        return False
    return tag in value

def is_asset(value):
    return is_tagged_dict('__mediachain_asset__', value)

def is_mediachain_object(value):
    return is_tagged_dict('__mediachain_object__', value)

def is_canonical(obj):
    return obj['type'] == 'artefact' or obj['type'] == 'entity'

def merge_meta(obj, meta):
    merged = {'data': {}}
    merged.update(meta)
    if 'meta' in obj:
        merged.update(obj['meta'])
    obj['meta'] = merged

def string_ref_to_map(string_ref):
    return {
        '@link': b58decode(string_ref)
    }

# TODO(yusef): replace string parsing hack with grpc error metadata
# requires support in grpc-java (in master but not yet released)

# this regex will only work for base58 encoded sha2-256 hashes,
# and will break if we ever change the error message :(
duplicate_insert_pattern = re.compile(
    '.*Duplicate Journal Insert.*(Qm[1-9a-km-zA-HJ-NP-Z]{44}).*'
)

def duplicate_canonical_ref(grpc_error_str):
    m = re.match(duplicate_insert_pattern, grpc_error_str)
    if not m:
        return None
    return m.group(1)

<commit_msg>use "raw datastore" for thumbs and raw_ref (works with ipfs)<commit_after>from __future__ import unicode_literals

from copy import deepcopy
import re
from base58 import b58decode
from mediachain.ingestion.asset_loader import load_asset, process_asset
from mediachain.datastore import get_raw_datastore
from mediachain.datastore.dynamo import get_db
from mediachain.datastore.data_objects import MultihashReference
from grpc.framework.interfaces.face.face import NetworkError

class Writer(object):
    def __init__(self, transactor, datastore=None,
                 download_remote_assets=False):
        self.transactor = transactor
        self.datastore = datastore or get_db()
        self.download_remote_assets = download_remote_assets

    def write_dataset(self, dataset_iterator):
        translator_id = dataset_iterator.translator.translator_id()

        for result in dataset_iterator:
            translated = result['translated']
            raw = result['raw_content']
            local_assets = result.get('local_assets', {})
            refs = self.submit_translator_output(translator_id, translated,
                                                 raw, local_assets)
            print('Inserted canonical: {}'.format(refs['canonical']))

    def submit_translator_output(self,
                                 translator_id,
                                 translated,
                                 raw_content,
                                 local_assets):
        meta_source = MultihashReference.from_base58(
            self.datastore.put(raw_content))

        raw_ref = self.store_raw(raw_content)

        common_meta = {'translator': translator_id,
                       'raw_ref': raw_ref.to_map()}

        canonical = translated['canonical']
        canonical = self.flatten_record(canonical, common_meta,
                                        meta_source, local_assets)
        canonical_ref = self.submit_object(canonical)

        chain = translated.get('chain', [])
        chain_refs = []
        for cell in chain:
            cell = self.flatten_record(cell, common_meta,
                                       meta_source, local_assets)
            cell['ref'] = canonical_ref.to_map()
            chain_ref = self.submit_object(cell)
            chain_refs += chain_ref.multihash_base58()
        return {
            'canonical': canonical_ref.multihash_base58(),
            'chain': chain_refs
        }

    def flatten_record(self, record, common_meta, meta_source, local_assets):
        record = deepcopy(record)
        assets = {k: v for k, v in record.iteritems()
                  if is_asset(v)}

        for k, a in assets.iteritems():
            local = local_assets.get(k, None)
            ref = self.store_asset(k, local, a)
            if ref is None:
                print('Unable to store asset with key {}, removing'.format(k))
                del record[k]
            else:
                record[k] = ref.to_map()

        objects = {k: v for k, v in record.iteritems()
                   if isinstance(v, dict)}

        for k, o in objects.iteritems():
            flat = self.flatten_record(o, common_meta,
                                       meta_source, local_assets)
            if is_mediachain_object(o):
                ref = self.submit_object(flat)
                record[k] = ref.to_map()
            else:
                record[k] = flat

        if is_mediachain_object(record):
            del record['__mediachain_object__']
            record['metaSource'] = meta_source.to_map()
            record['meta'].update(common_meta)
        return record

    def store_raw(self, raw_content):
        store = get_raw_datastore()
        ref = store.put(raw_content)
        return MultihashReference.from_base58(ref)

    def store_asset(self, name, local_asset, remote_asset):
        data = load_asset(local_asset)
        if data is None and self.download_remote_assets:
            data = load_asset(remote_asset)
        if data is None:
            return None
        data = process_asset(name, data)
        return self.store_raw(data)

    def submit_object(self, obj):
        try:
            if is_canonical(obj):
                return self.transactor.insert_canonical(obj)
            else:
                return self.transactor.update_chain(obj)
        except NetworkError as e:
            ref_str = duplicate_canonical_ref(e.details)
            if ref_str is None:
                raise

        return MultihashReference.from_base58(ref_str)

def is_tagged_dict(tag, value):
    if not isinstance(value, dict):
        return False
    return tag in value

def is_asset(value):
    return is_tagged_dict('__mediachain_asset__', value)

def is_mediachain_object(value):
    return is_tagged_dict('__mediachain_object__', value)

def is_canonical(obj):
    return obj['type'] == 'artefact' or obj['type'] == 'entity'

def merge_meta(obj, meta):
    merged = {'data': {}}
    merged.update(meta)
    if 'meta' in obj:
        merged.update(obj['meta'])
    obj['meta'] = merged

def string_ref_to_map(string_ref):
    return {
        '@link': b58decode(string_ref)
    }

# TODO(yusef): replace string parsing hack with grpc error metadata
# requires support in grpc-java (in master but not yet released)

# this regex will only work for base58 encoded sha2-256 hashes,
# and will break if we ever change the error message :(
duplicate_insert_pattern = re.compile(
    '.*Duplicate Journal Insert.*(Qm[1-9a-km-zA-HJ-NP-Z]{44}).*'
)

def duplicate_canonical_ref(grpc_error_str):
    m = re.match(duplicate_insert_pattern, grpc_error_str)
    if not m:
        return None
    return m.group(1)

<|endoftext|>


<commit_before>###############################################################################
#
# The MIT License (MIT)
#
# Copyright (c) Crossbar.io Technologies GmbH
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.
#
###############################################################################

from autobahn.twisted.component import Component, run
from autobahn.twisted.util import sleep
from twisted.internet.defer import inlineCallbacks
import os

url = os.environ.get('CBURL', u'ws://localhost:8080/ws')
realmvalue = os.environ.get('CBREALM', u'realm1')
component = Component(transports=url, realm=realmvalue)

@component.on_join
@inlineCallbacks
def joined(session, details):
    print("session ready")

    def oncounter(count):
        print("event received: {0}", count)

    try:
        yield session.subscribe(oncounter, u'my.com.hello')
        print("subscribed to topic")
    except Exception as e:
        print("could not subscribe to topic: {0}".format(e))

if __name__ == "__main__":
    run([component])

<commit_msg>Add option to pass topic as arg<commit_after>###############################################################################
#
# The MIT License (MIT)
#
# Copyright (c) Crossbar.io Technologies GmbH
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.
#
###############################################################################

from autobahn.twisted.component import Component, run
from autobahn.twisted.util import sleep
from twisted.internet.defer import inlineCallbacks
import os

url = os.environ.get('CBURL', u'ws://localhost:8080/ws')
realmvalue = os.environ.get('CBREALM', u'realm1')
topic  = os.environ.get('CBTOPIC', u'my.com.hello')
component = Component(transports=url, realm=realmvalue)

@component.on_join
@inlineCallbacks
def joined(session, details):
    print("session ready")

    def oncounter(count):
        print("event received: {0}", count)

    try:
        yield session.subscribe(oncounter, u'my.com.hello')
        print("subscribed to topic")
    except Exception as e:
        print("could not subscribe to topic: {0}".format(e))

if __name__ == "__main__":
    run([component])

<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
##
## This file is part of Invenio.
## Copyright (C) 2013 CERN.
##
## Invenio is free software; you can redistribute it and/or
## modify it under the terms of the GNU General Public License as
## published by the Free Software Foundation; either version 2 of the
## License, or (at your option) any later version.
##
## Invenio is distributed in the hope that it will be useful, but
## WITHOUT ANY WARRANTY; without even the implied warranty of
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
## General Public License for more details.
##
## You should have received a copy of the GNU General Public License
## along with Invenio; if not, write to the Free Software Foundation, Inc.,
## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.

""" Plugin to correct and validate dates """

from datetime import datetime

try:
    from dateutil import parser
    HAS_DATEUTIL = True
except ImportError:
    HAS_DATEUTIL = False

def check_record(record, fields, dayfirst=True, yearfirst=False,
        date_format="%Y-%m-%d", allow_future=True,
        minimum_date=datetime(1800,1,1)):
    """
    Corrects and validates date fields

    For detailed explanation of how dayfirst and yearfirst works, visit
    http://labix.org/python-dateutil#head-b95ce2094d189a89f80f5ae52a05b4ab7b41af47

    For detailed explanation of the date_format placeholders, visit
    http://docs.python.org/2/library/datetime.html#strftime-strptime-behavior

    This plugin needs the python-dateutil library to work.

    @param dayfirst Consider the day first if ambiguous
    @type dayfirst boolean

    @param yearfirst Consider year first if ambiguous
    @type yearfirst boolean

    @param date_format normalized date format
    @type date_format string

    @param allow_future If False, dates in the future will be marked as invalid
    @type allow_future boolean

    @param minimum_date dates older than this will be rejected. Default Jan 1 1800
    @type minimum_date datetime.datetime
    """
    if not HAS_DATEUTIL:
        return

    for position, value in record.iterfields(fields):
        try:
            new_date = parser.parse(value, dayfirst=dayfirst, yearfirst=yearfirst)
        except ValueError:
            record.set_invalid("Non-parseable date format in field %s" % position[0])
            continue

        if not allow_future and new_date > datetime.now():
            record.set_invalid("Date in the future in field %s" % position[0])

        if new_date < minimum_date:
            record.set_invalid("Date too old (less than minimum_date) in field %s" % position[0])

        if new_date < datetime(1900, 1, 1):
            continue # strftime doesn't accept older dates

        new_date_str = new_date.strftime(date_format)
        record.amend_field(position, new_date_str)

<commit_msg>BibCheck: compatiblity with dateutil 2.2<commit_after># -*- coding: utf-8 -*-
##
## This file is part of Invenio.
## Copyright (C) 2013 CERN.
##
## Invenio is free software; you can redistribute it and/or
## modify it under the terms of the GNU General Public License as
## published by the Free Software Foundation; either version 2 of the
## License, or (at your option) any later version.
##
## Invenio is distributed in the hope that it will be useful, but
## WITHOUT ANY WARRANTY; without even the implied warranty of
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
## General Public License for more details.
##
## You should have received a copy of the GNU General Public License
## along with Invenio; if not, write to the Free Software Foundation, Inc.,
## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.

""" Plugin to correct and validate dates """

from datetime import datetime

try:
    from dateutil import parser
    HAS_DATEUTIL = True
except ImportError:
    HAS_DATEUTIL = False

def check_record(record, fields, dayfirst=True, yearfirst=False,
        date_format="%Y-%m-%d", allow_future=True,
        minimum_date=datetime(1800,1,1)):
    """
    Corrects and validates date fields

    For detailed explanation of how dayfirst and yearfirst works, visit
    http://labix.org/python-dateutil#head-b95ce2094d189a89f80f5ae52a05b4ab7b41af47

    For detailed explanation of the date_format placeholders, visit
    http://docs.python.org/2/library/datetime.html#strftime-strptime-behavior

    This plugin needs the python-dateutil library to work.

    @param dayfirst Consider the day first if ambiguous
    @type dayfirst boolean

    @param yearfirst Consider year first if ambiguous
    @type yearfirst boolean

    @param date_format normalized date format
    @type date_format string

    @param allow_future If False, dates in the future will be marked as invalid
    @type allow_future boolean

    @param minimum_date dates older than this will be rejected. Default Jan 1 1800
    @type minimum_date datetime.datetime
    """
    if not HAS_DATEUTIL:
        return

    for position, value in record.iterfields(fields):
        try:
            new_date = parser.parse(value, dayfirst=dayfirst, yearfirst=yearfirst)
        except (ValueError, TypeError):
            record.set_invalid("Non-parseable date format in field %s" % position[0])
            continue

        if not allow_future and new_date > datetime.now():
            record.set_invalid("Date in the future in field %s" % position[0])

        if new_date < minimum_date:
            record.set_invalid("Date too old (less than minimum_date) in field %s" % position[0])

        if new_date < datetime(1900, 1, 1):
            continue # strftime doesn't accept older dates

        new_date_str = new_date.strftime(date_format)
        record.amend_field(position, new_date_str)

<|endoftext|>