<commit_before>"""Pyramid request argument parsing.

Example usage: ::

    from wsgiref.simple_server import make_server
    from pyramid.config import Configurator
    from pyramid.response import Response
    from marshmallow import fields
    from webargs.pyramidparser import use_args

    hello_args = {
        'name': fields.Str(missing='World')
    }

    @use_args(hello_args)
    def hello_world(request, args):
        return Response('Hello ' + args['name'])

    if __name__ == '__main__':
        config = Configurator()
        config.add_route('hello', '/')
        config.add_view(hello_world, route_name='hello')
        app = config.make_wsgi_app()
        server = make_server('0.0.0.0', 6543, app)
        server.serve_forever()
"""
import collections
import functools

from webob.multidict import MultiDict
from pyramid.httpexceptions import exception_response

from webargs import core
from webargs.core import json
from webargs.multidictproxy import MultiDictProxy

def is_json_request(req):
    return core.is_json(req.headers.get("content-type"))

class PyramidParser(core.Parser):
    """Pyramid request argument parser."""

    __location_map__ = dict(
        matchdict="load_matchdict",
        path="load_matchdict",
        **core.Parser.__location_map__,
    )

    def _raw_load_json(self, req):
        """Return a json payload from the request for the core parser's load_json

        Checks the input mimetype and may return 'missing' if the mimetype is
        non-json, even if the request body is parseable as json."""
        if not is_json_request(req):
            return core.missing

        return core.parse_json(req.body, req.charset)

    def load_querystring(self, req, schema):
        """Return query params from the request as a MultiDictProxy."""
        return MultiDictProxy(req.GET, schema)

    def load_form(self, req, schema):
        """Return form values from the request as a MultiDictProxy."""
        return MultiDictProxy(req.POST, schema)

    def load_cookies(self, req, schema):
        """Return cookies from the request as a MultiDictProxy."""
        return MultiDictProxy(req.cookies, schema)

    def load_headers(self, req, schema):
        """Return headers from the request as a MultiDictProxy."""
        return MultiDictProxy(req.headers, schema)

    def load_files(self, req, schema):
        """Return files from the request as a MultiDictProxy."""
        files = ((k, v) for k, v in req.POST.items() if hasattr(v, "file"))
        return MultiDictProxy(MultiDict(files), schema)

    def load_matchdict(self, req, schema):
        """Return the request's ``matchdict`` as a MultiDictProxy."""
        return MultiDictProxy(req.matchdict, schema)

    def handle_error(self, error, req, schema, error_status_code, error_headers):
        """Handles errors during parsing. Aborts the current HTTP request and
        responds with a 400 error.
        """
        status_code = error_status_code or self.DEFAULT_VALIDATION_STATUS
        response = exception_response(
            status_code,
            detail=str(error),
            headers=error_headers,
            content_type="application/json",
        )
        body = json.dumps(error.messages)
        response.body = body.encode("utf-8") if isinstance(body, str) else body
        raise response

    def _handle_invalid_json_error(self, error, req, *args, **kwargs):
        messages = {"json": ["Invalid JSON body."]}
        response = exception_response(
            400, detail=str(messages), content_type="application/json"
        )
        body = json.dumps(messages)
        response.body = body.encode("utf-8") if isinstance(body, str) else body
        raise response

    def use_args(
        self,
        argmap,
        req=None,
        location=core.Parser.DEFAULT_LOCATION,
        as_kwargs=False,
        validate=None,
        error_status_code=None,
        error_headers=None,
    ):
        """Decorator that injects parsed arguments into a view callable.
        Supports the *Class-based View* pattern where `request` is saved as an instance
        attribute on a view class.

        :param dict argmap: Either a `marshmallow.Schema`, a `dict`
            of argname -> `marshmallow.fields.Field` pairs, or a callable
            which accepts a request and returns a `marshmallow.Schema`.
        :param req: The request object to parse. Pulled off of the view by default.
        :param str location: Where on the request to load values.
        :param bool as_kwargs: Whether to insert arguments as keyword arguments.
        :param callable validate: Validation function that receives the dictionary
            of parsed arguments. If the function returns ``False``, the parser
            will raise a :exc:`ValidationError`.
        :param int error_status_code: Status code passed to error handler functions when
            a `ValidationError` is raised.
        :param dict error_headers: Headers passed to error handler functions when a
            a `ValidationError` is raised.
        """
        location = location or self.location
        # Optimization: If argmap is passed as a dictionary, we only need
        # to generate a Schema once
        if isinstance(argmap, collections.Mapping):
            argmap = core.dict2schema(argmap, self.schema_class)()

        def decorator(func):
            @functools.wraps(func)
            def wrapper(obj, *args, **kwargs):
                # The first argument is either `self` or `request`
                try:  # get self.request
                    request = req or obj.request
                except AttributeError:  # first arg is request
                    request = obj
                # NOTE: At this point, argmap may be a Schema, callable, or dict
                parsed_args = self.parse(
                    argmap,
                    req=request,
                    location=location,
                    validate=validate,
                    error_status_code=error_status_code,
                    error_headers=error_headers,
                )
                if as_kwargs:
                    kwargs.update(parsed_args)
                    return func(obj, *args, **kwargs)
                else:
                    return func(obj, parsed_args, *args, **kwargs)

            wrapper.__wrapped__ = func
            return wrapper

        return decorator

parser = PyramidParser()
use_args = parser.use_args
use_kwargs = parser.use_kwargs
<commit_msg>Import ABC from collections.abc instead of collections for Python 3.9 compatibility.<commit_after>"""Pyramid request argument parsing.

Example usage: ::

    from wsgiref.simple_server import make_server
    from pyramid.config import Configurator
    from pyramid.response import Response
    from marshmallow import fields
    from webargs.pyramidparser import use_args

    hello_args = {
        'name': fields.Str(missing='World')
    }

    @use_args(hello_args)
    def hello_world(request, args):
        return Response('Hello ' + args['name'])

    if __name__ == '__main__':
        config = Configurator()
        config.add_route('hello', '/')
        config.add_view(hello_world, route_name='hello')
        app = config.make_wsgi_app()
        server = make_server('0.0.0.0', 6543, app)
        server.serve_forever()
"""
import collections
import functools

from webob.multidict import MultiDict
from pyramid.httpexceptions import exception_response

from webargs import core
from webargs.core import json
from webargs.multidictproxy import MultiDictProxy

def is_json_request(req):
    return core.is_json(req.headers.get("content-type"))

class PyramidParser(core.Parser):
    """Pyramid request argument parser."""

    __location_map__ = dict(
        matchdict="load_matchdict",
        path="load_matchdict",
        **core.Parser.__location_map__,
    )

    def _raw_load_json(self, req):
        """Return a json payload from the request for the core parser's load_json

        Checks the input mimetype and may return 'missing' if the mimetype is
        non-json, even if the request body is parseable as json."""
        if not is_json_request(req):
            return core.missing

        return core.parse_json(req.body, req.charset)

    def load_querystring(self, req, schema):
        """Return query params from the request as a MultiDictProxy."""
        return MultiDictProxy(req.GET, schema)

    def load_form(self, req, schema):
        """Return form values from the request as a MultiDictProxy."""
        return MultiDictProxy(req.POST, schema)

    def load_cookies(self, req, schema):
        """Return cookies from the request as a MultiDictProxy."""
        return MultiDictProxy(req.cookies, schema)

    def load_headers(self, req, schema):
        """Return headers from the request as a MultiDictProxy."""
        return MultiDictProxy(req.headers, schema)

    def load_files(self, req, schema):
        """Return files from the request as a MultiDictProxy."""
        files = ((k, v) for k, v in req.POST.items() if hasattr(v, "file"))
        return MultiDictProxy(MultiDict(files), schema)

    def load_matchdict(self, req, schema):
        """Return the request's ``matchdict`` as a MultiDictProxy."""
        return MultiDictProxy(req.matchdict, schema)

    def handle_error(self, error, req, schema, error_status_code, error_headers):
        """Handles errors during parsing. Aborts the current HTTP request and
        responds with a 400 error.
        """
        status_code = error_status_code or self.DEFAULT_VALIDATION_STATUS
        response = exception_response(
            status_code,
            detail=str(error),
            headers=error_headers,
            content_type="application/json",
        )
        body = json.dumps(error.messages)
        response.body = body.encode("utf-8") if isinstance(body, str) else body
        raise response

    def _handle_invalid_json_error(self, error, req, *args, **kwargs):
        messages = {"json": ["Invalid JSON body."]}
        response = exception_response(
            400, detail=str(messages), content_type="application/json"
        )
        body = json.dumps(messages)
        response.body = body.encode("utf-8") if isinstance(body, str) else body
        raise response

    def use_args(
        self,
        argmap,
        req=None,
        location=core.Parser.DEFAULT_LOCATION,
        as_kwargs=False,
        validate=None,
        error_status_code=None,
        error_headers=None,
    ):
        """Decorator that injects parsed arguments into a view callable.
        Supports the *Class-based View* pattern where `request` is saved as an instance
        attribute on a view class.

        :param dict argmap: Either a `marshmallow.Schema`, a `dict`
            of argname -> `marshmallow.fields.Field` pairs, or a callable
            which accepts a request and returns a `marshmallow.Schema`.
        :param req: The request object to parse. Pulled off of the view by default.
        :param str location: Where on the request to load values.
        :param bool as_kwargs: Whether to insert arguments as keyword arguments.
        :param callable validate: Validation function that receives the dictionary
            of parsed arguments. If the function returns ``False``, the parser
            will raise a :exc:`ValidationError`.
        :param int error_status_code: Status code passed to error handler functions when
            a `ValidationError` is raised.
        :param dict error_headers: Headers passed to error handler functions when a
            a `ValidationError` is raised.
        """
        location = location or self.location
        # Optimization: If argmap is passed as a dictionary, we only need
        # to generate a Schema once
        if isinstance(argmap, collections.abc.Mapping):
            argmap = core.dict2schema(argmap, self.schema_class)()

        def decorator(func):
            @functools.wraps(func)
            def wrapper(obj, *args, **kwargs):
                # The first argument is either `self` or `request`
                try:  # get self.request
                    request = req or obj.request
                except AttributeError:  # first arg is request
                    request = obj
                # NOTE: At this point, argmap may be a Schema, callable, or dict
                parsed_args = self.parse(
                    argmap,
                    req=request,
                    location=location,
                    validate=validate,
                    error_status_code=error_status_code,
                    error_headers=error_headers,
                )
                if as_kwargs:
                    kwargs.update(parsed_args)
                    return func(obj, *args, **kwargs)
                else:
                    return func(obj, parsed_args, *args, **kwargs)

            wrapper.__wrapped__ = func
            return wrapper

        return decorator

parser = PyramidParser()
use_args = parser.use_args
use_kwargs = parser.use_kwargs
<|endoftext|>


<commit_before>"""
This module controls defines celery tasks and their applicable schedules. The celery beat server and workers will start
when invoked.

When ran in development mode (LEMUR_CONFIG=<location of development configuration file. To run both the celery
beat scheduler and a worker simultaneously, and to have jobs kick off starting at the next minute, run the following
command: celery -A lemur.common.celery worker --loglevel=info -l DEBUG -B

"""
import copy
import sys
from datetime import datetime, timezone, timedelta

from celery import Celery
from flask import current_app

from lemur.authorities.service import get as get_authority
from lemur.factory import create_app
from lemur.notifications.messaging import send_pending_failure_notification
from lemur.pending_certificates import service as pending_certificate_service
from lemur.plugins.base import plugins, IPlugin
from lemur.sources.cli import clean, sync, validate_sources
from lemur.destinations import service as destinations_service
from lemur.sources import service as sources_service

if current_app:
    flask_app = current_app
else:
    flask_app = create_app()

def make_celery(app):
    celery = Celery(app.import_name, backend=app.config.get('CELERY_RESULT_BACKEND'),
                    broker=app.config.get('CELERY_BROKER_URL'))
    celery.conf.update(app.config)
    TaskBase = celery.Task

    class ContextTask(TaskBase):
        abstract = True

        def __call__(self, *args, **kwargs):
            with app.app_context():
                return TaskBase.__call__(self, *args, **kwargs)

    celery.Task = ContextTask
    return celery

celery = make_celery(flask_app)

def is_task_active(fun, task_id, args):
    from celery.task.control import inspect
    i = inspect()
    active_tasks = i.active()
    for _, tasks in active_tasks.items():
        for task in tasks:
            if task.get("id") == task_id:
                continue
            if task.get("name") == fun and task.get("args") == str(args):
                return True
    return False

@celery.task()
def fetch_acme_cert(id):
    """
    Attempt to get the full certificate for the pending certificate listed.

    Args:
        id: an id of a PendingCertificate
    """
    log_data = {
        "function": "{}.{}".format(__name__, sys._getframe().f_code.co_name),
        "message": "Resolving pending certificate {}".format(id)
    }
    current_app.logger.debug(log_data)
    pending_certs = pending_certificate_service.get_pending_certs([id])
    new = 0
    failed = 0
    wrong_issuer = 0
    acme_certs = []

    # We only care about certs using the acme-issuer plugin
    for cert in pending_certs:
        cert_authority = get_authority(cert.authority_id)
        if cert_authority.plugin_name == 'acme-issuer':
            acme_certs.append(cert)
        else:
            wrong_issuer += 1

    authority = plugins.get("acme-issuer")
    resolved_certs = authority.get_ordered_certificates(acme_certs)

    for cert in resolved_certs:
        real_cert = cert.get("cert")
        # It's necessary to reload the pending cert due to detached instance: http://sqlalche.me/e/bhk3
        pending_cert = pending_certificate_service.get(cert.get("pending_cert").id)
        if not pending_cert:
            log_data["message"] = "Pending certificate doesn't exist anymore. Was it resolved by another process?"
            current_app.logger.error(log_data)
            continue
        if real_cert:
            # If a real certificate was returned from issuer, then create it in Lemur and mark
            # the pending certificate as resolved
            final_cert = pending_certificate_service.create_certificate(pending_cert, real_cert, pending_cert.user)
            pending_certificate_service.update(
                cert.get("pending_cert").id,
                resolved=True
            )
            pending_certificate_service.update(
                cert.get("pending_cert").id,
                resolved_cert_id=final_cert.id
            )
            # add metrics to metrics extension
            new += 1
        else:
            failed += 1
            error_log = copy.deepcopy(log_data)
            error_log["message"] = "Pending certificate creation failure"
            error_log["pending_cert_id"] = pending_cert.id
            error_log["last_error"] = cert.get("last_error")
            error_log["cn"] = pending_cert.cn

            if pending_cert.number_attempts > 4:
                error_log["message"] = "Deleting pending certificate"
                send_pending_failure_notification(pending_cert, notify_owner=pending_cert.notify)
                # Mark the pending cert as resolved
                pending_certificate_service.update(
                    cert.get("pending_cert").id,
                    resolved=True
                )
            else:
                pending_certificate_service.increment_attempt(pending_cert)
                pending_certificate_service.update(
                    cert.get("pending_cert").id,
                    status=str(cert.get("last_error"))
                )
                # Add failed pending cert task back to queue
                fetch_acme_cert.delay(id)
            current_app.logger.error(error_log)
    log_data["message"] = "Complete"
    log_data["new"] = new
    log_data["failed"] = failed
    log_data["wrong_issuer"] = wrong_issuer
    current_app.logger.debug(log_data)
    print(
        "[+] Certificates: New: {new} Failed: {failed} Not using ACME: {wrong_issuer}".format(
            new=new,
            failed=failed,
            wrong_issuer=wrong_issuer
        )
    )

@celery.task()
def fetch_all_pending_acme_certs():
    """Instantiate celery workers to resolve all pending Acme certificates"""
    pending_certs = pending_certificate_service.get_unresolved_pending_certs()

    log_data = {
        "function": "{}.{}".format(__name__, sys._getframe().f_code.co_name),
        "message": "Starting job."
    }

    current_app.logger.debug(log_data)

    # We only care about certs using the acme-issuer plugin
    for cert in pending_certs:
        cert_authority = get_authority(cert.authority_id)
        if cert_authority.plugin_name == 'acme-issuer':
            if datetime.now(timezone.utc) - cert.last_updated > timedelta(minutes=5):
                log_data["message"] = "Triggering job for cert {}".format(cert.name)
                log_data["cert_name"] = cert.name
                log_data["cert_id"] = cert.id
                current_app.logger.debug(log_data)
                fetch_acme_cert.delay(cert.id)

@celery.task()
def remove_old_acme_certs():
    """Prune old pending acme certificates from the database"""
    log_data = {
        "function": "{}.{}".format(__name__, sys._getframe().f_code.co_name)
    }
    pending_certs = pending_certificate_service.get_pending_certs('all')

    # Delete pending certs more than a week old
    for cert in pending_certs:
        if datetime.now(timezone.utc) - cert.last_updated > timedelta(days=7):
            log_data['pending_cert_id'] = cert.id
            log_data['pending_cert_name'] = cert.name
            log_data['message'] = "Deleting pending certificate"
            current_app.logger.debug(log_data)
            pending_certificate_service.delete(cert.id)

@celery.task()
def clean_all_sources():
    """
    This function will clean unused certificates from sources. This is a destructive operation and should only
    be ran periodically. This function triggers one celery task per source.
    """
    sources = validate_sources("all")
    for source in sources:
        current_app.logger.debug("Creating celery task to clean source {}".format(source.label))
        clean_source.delay(source.label)

@celery.task()
def clean_source(source):
    """
    This celery task will clean the specified source. This is a destructive operation that will delete unused
    certificates from each source.

    :param source:
    :return:
    """
    current_app.logger.debug("Cleaning source {}".format(source))
    clean([source], True)

@celery.task()
def sync_all_sources():
    """
    This function will sync certificates from all sources. This function triggers one celery task per source.
    """
    sources = validate_sources("all")
    for source in sources:
        current_app.logger.debug("Creating celery task to sync source {}".format(source.label))
        sync_source.delay(source.label)

@celery.task()
def sync_source(source):
    """
    This celery task will sync the specified source.

    :param source:
    :return:
    """

    function = "{}.{}".format(__name__, sys._getframe().f_code.co_name)
    task_id = celery.current_task.request.id
    log_data = {
        "function": function,
        "message": "Syncing source",
        "source": source,
        "task_id": task_id,
    }
    current_app.logger.debug(log_data)

    if is_task_active(function, task_id, (source,)):
        log_data["message"] = "Skipping task: Task is already active"
        current_app.logger.debug(log_data)
        return
    sync([source])
    log_data["message"] = "Done syncing source"
    current_app.logger.debug(log_data)

@celery.task()
def sync_source_destination():
    """
    This celery task will sync destination and source, to make sure all new destinations are also present as source.
    Some destinations do not qualify as sources, and hence should be excluded from being added as sources
    We identify qualified destinations based on the sync_as_source attributed of the plugin.
    The destination sync_as_source_name reviels the name of the suitable source-plugin.
    We rely on account numbers to avoid duplicates.
    """
    current_app.logger.debug("Syncing source and destination")

    # a set of all accounts numbers available as sources
    src_accounts = set()
    sources = validate_sources("all")
    for src in sources:
        src_accounts.add(IPlugin.get_option('accountNumber', src.options))

    for dst in destinations_service.get_all():
        destination_plugin = plugins.get(dst.plugin_name)
        account_number = IPlugin.get_option('accountNumber', dst.options)
        if destination_plugin.sync_as_source and (account_number not in src_accounts):
            src_options = copy.deepcopy(plugins.get(destination_plugin.sync_as_source_name).options)
            for o in src_options:
                if o.get('name') == 'accountNumber':
                    o.update({'value': account_number})
            sources_service.create(label=dst.label,
                                   plugin_name=destination_plugin.sync_as_source_name,
                                   options=src_options,
                                   description=dst.description)
            current_app.logger.info("Source: %s added", dst.label)

<commit_msg>removing lines<commit_after>"""
This module controls defines celery tasks and their applicable schedules. The celery beat server and workers will start
when invoked.

When ran in development mode (LEMUR_CONFIG=<location of development configuration file. To run both the celery
beat scheduler and a worker simultaneously, and to have jobs kick off starting at the next minute, run the following
command: celery -A lemur.common.celery worker --loglevel=info -l DEBUG -B

"""
import copy
import sys
from datetime import datetime, timezone, timedelta

from celery import Celery
from flask import current_app

from lemur.authorities.service import get as get_authority
from lemur.factory import create_app
from lemur.notifications.messaging import send_pending_failure_notification
from lemur.pending_certificates import service as pending_certificate_service
from lemur.plugins.base import plugins, IPlugin
from lemur.sources.cli import clean, sync, validate_sources
from lemur.destinations import service as destinations_service
from lemur.sources import service as sources_service

if current_app:
    flask_app = current_app
else:
    flask_app = create_app()

def make_celery(app):
    celery = Celery(app.import_name, backend=app.config.get('CELERY_RESULT_BACKEND'),
                    broker=app.config.get('CELERY_BROKER_URL'))
    celery.conf.update(app.config)
    TaskBase = celery.Task

    class ContextTask(TaskBase):
        abstract = True

        def __call__(self, *args, **kwargs):
            with app.app_context():
                return TaskBase.__call__(self, *args, **kwargs)

    celery.Task = ContextTask
    return celery

celery = make_celery(flask_app)

def is_task_active(fun, task_id, args):
    from celery.task.control import inspect
    i = inspect()
    active_tasks = i.active()
    for _, tasks in active_tasks.items():
        for task in tasks:
            if task.get("id") == task_id:
                continue
            if task.get("name") == fun and task.get("args") == str(args):
                return True
    return False

@celery.task()
def fetch_acme_cert(id):
    """
    Attempt to get the full certificate for the pending certificate listed.

    Args:
        id: an id of a PendingCertificate
    """
    log_data = {
        "function": "{}.{}".format(__name__, sys._getframe().f_code.co_name),
        "message": "Resolving pending certificate {}".format(id)
    }
    current_app.logger.debug(log_data)
    pending_certs = pending_certificate_service.get_pending_certs([id])
    new = 0
    failed = 0
    wrong_issuer = 0
    acme_certs = []

    # We only care about certs using the acme-issuer plugin
    for cert in pending_certs:
        cert_authority = get_authority(cert.authority_id)
        if cert_authority.plugin_name == 'acme-issuer':
            acme_certs.append(cert)
        else:
            wrong_issuer += 1

    authority = plugins.get("acme-issuer")
    resolved_certs = authority.get_ordered_certificates(acme_certs)

    for cert in resolved_certs:
        real_cert = cert.get("cert")
        # It's necessary to reload the pending cert due to detached instance: http://sqlalche.me/e/bhk3
        pending_cert = pending_certificate_service.get(cert.get("pending_cert").id)
        if not pending_cert:
            log_data["message"] = "Pending certificate doesn't exist anymore. Was it resolved by another process?"
            current_app.logger.error(log_data)
            continue
        if real_cert:
            # If a real certificate was returned from issuer, then create it in Lemur and mark
            # the pending certificate as resolved
            final_cert = pending_certificate_service.create_certificate(pending_cert, real_cert, pending_cert.user)
            pending_certificate_service.update(
                cert.get("pending_cert").id,
                resolved=True
            )
            pending_certificate_service.update(
                cert.get("pending_cert").id,
                resolved_cert_id=final_cert.id
            )
            # add metrics to metrics extension
            new += 1
        else:
            failed += 1
            error_log = copy.deepcopy(log_data)
            error_log["message"] = "Pending certificate creation failure"
            error_log["pending_cert_id"] = pending_cert.id
            error_log["last_error"] = cert.get("last_error")
            error_log["cn"] = pending_cert.cn

            if pending_cert.number_attempts > 4:
                error_log["message"] = "Deleting pending certificate"
                send_pending_failure_notification(pending_cert, notify_owner=pending_cert.notify)
                # Mark the pending cert as resolved
                pending_certificate_service.update(
                    cert.get("pending_cert").id,
                    resolved=True
                )
            else:
                pending_certificate_service.increment_attempt(pending_cert)
                pending_certificate_service.update(
                    cert.get("pending_cert").id,
                    status=str(cert.get("last_error"))
                )
                # Add failed pending cert task back to queue
                fetch_acme_cert.delay(id)
            current_app.logger.error(error_log)
    log_data["message"] = "Complete"
    log_data["new"] = new
    log_data["failed"] = failed
    log_data["wrong_issuer"] = wrong_issuer
    current_app.logger.debug(log_data)
    print(
        "[+] Certificates: New: {new} Failed: {failed} Not using ACME: {wrong_issuer}".format(
            new=new,
            failed=failed,
            wrong_issuer=wrong_issuer
        )
    )

@celery.task()
def fetch_all_pending_acme_certs():
    """Instantiate celery workers to resolve all pending Acme certificates"""
    pending_certs = pending_certificate_service.get_unresolved_pending_certs()

    log_data = {
        "function": "{}.{}".format(__name__, sys._getframe().f_code.co_name),
        "message": "Starting job."
    }

    current_app.logger.debug(log_data)

    # We only care about certs using the acme-issuer plugin
    for cert in pending_certs:
        cert_authority = get_authority(cert.authority_id)
        if cert_authority.plugin_name == 'acme-issuer':
            if datetime.now(timezone.utc) - cert.last_updated > timedelta(minutes=5):
                log_data["message"] = "Triggering job for cert {}".format(cert.name)
                log_data["cert_name"] = cert.name
                log_data["cert_id"] = cert.id
                current_app.logger.debug(log_data)
                fetch_acme_cert.delay(cert.id)

@celery.task()
def remove_old_acme_certs():
    """Prune old pending acme certificates from the database"""
    log_data = {
        "function": "{}.{}".format(__name__, sys._getframe().f_code.co_name)
    }
    pending_certs = pending_certificate_service.get_pending_certs('all')

    # Delete pending certs more than a week old
    for cert in pending_certs:
        if datetime.now(timezone.utc) - cert.last_updated > timedelta(days=7):
            log_data['pending_cert_id'] = cert.id
            log_data['pending_cert_name'] = cert.name
            log_data['message'] = "Deleting pending certificate"
            current_app.logger.debug(log_data)
            pending_certificate_service.delete(cert.id)

@celery.task()
def clean_all_sources():
    """
    This function will clean unused certificates from sources. This is a destructive operation and should only
    be ran periodically. This function triggers one celery task per source.
    """
    sources = validate_sources("all")
    for source in sources:
        current_app.logger.debug("Creating celery task to clean source {}".format(source.label))
        clean_source.delay(source.label)

@celery.task()
def clean_source(source):
    """
    This celery task will clean the specified source. This is a destructive operation that will delete unused
    certificates from each source.

    :param source:
    :return:
    """
    current_app.logger.debug("Cleaning source {}".format(source))
    clean([source], True)

@celery.task()
def sync_all_sources():
    """
    This function will sync certificates from all sources. This function triggers one celery task per source.
    """
    sources = validate_sources("all")
    for source in sources:
        current_app.logger.debug("Creating celery task to sync source {}".format(source.label))
        sync_source.delay(source.label)

@celery.task()
def sync_source(source):
    """
    This celery task will sync the specified source.

    :param source:
    :return:
    """

    function = "{}.{}".format(__name__, sys._getframe().f_code.co_name)
    task_id = celery.current_task.request.id
    log_data = {
        "function": function,
        "message": "Syncing source",
        "source": source,
        "task_id": task_id,
    }
    current_app.logger.debug(log_data)

    if is_task_active(function, task_id, (source,)):
        log_data["message"] = "Skipping task: Task is already active"
        current_app.logger.debug(log_data)
        return
    sync([source])
    log_data["message"] = "Done syncing source"
    current_app.logger.debug(log_data)

@celery.task()
def sync_source_destination():
    """
    This celery task will sync destination and source, to make sure all new destinations are also present as source.
    Some destinations do not qualify as sources, and hence should be excluded from being added as sources
    We identify qualified destinations based on the sync_as_source attributed of the plugin.
    The destination sync_as_source_name reviels the name of the suitable source-plugin.
    We rely on account numbers to avoid duplicates.
    """
    current_app.logger.debug("Syncing source and destination")

    # a set of all accounts numbers available as sources
    src_accounts = set()
    sources = validate_sources("all")
    for src in sources:
        src_accounts.add(IPlugin.get_option('accountNumber', src.options))

    for dst in destinations_service.get_all():
        destination_plugin = plugins.get(dst.plugin_name)
        account_number = IPlugin.get_option('accountNumber', dst.options)
        if destination_plugin.sync_as_source and (account_number not in src_accounts):
            src_options = copy.deepcopy(plugins.get(destination_plugin.sync_as_source_name).options)
            for o in src_options:
                if o.get('name') == 'accountNumber':
                    o.update({'value': account_number})
            sources_service.create(label=dst.label,
                                   plugin_name=destination_plugin.sync_as_source_name,
                                   options=src_options,
                                   description=dst.description)
            current_app.logger.info("Source: %s added", dst.label)
<|endoftext|>


<commit_before>import uuid
from django.test import SimpleTestCase
from django.test.utils import override_settings
from casexml.apps.case.mock import CaseFactory, CaseStructure, CaseRelationship
from casexml.apps.phone.cleanliness import set_cleanliness_flags, hint_still_valid, \
    get_cleanliness_flag_from_scratch
from casexml.apps.phone.data_providers.case.clean_owners import pop_ids
from casexml.apps.phone.models import OwnershipCleanlinessFlag
from casexml.apps.phone.tests.test_sync_mode import SyncBaseTest
from corehq.toggles import OWNERSHIP_CLEANLINESS

class OwnerCleanlinessTest(SyncBaseTest):

    def setUp(self):
        super(OwnerCleanlinessTest, self).setUp()
        # ensure that randomization is on
        OWNERSHIP_CLEANLINESS.randomness = 1
        self.owner_id = uuid.uuid4().hex
        self.synclog_id = uuid.uuid4().hex
        self.domain = uuid.uuid4().hex
        self.factory = CaseFactory(
            domain=self.domain,
            case_defaults={
                'create': True,
                'owner_id': self.owner_id,
                'user_id': self.owner_id,
            }
        )
        self.assert_owner_clean()  # this first call creates the OwnershipCleanliness doc
        self.sample_case = self.factory.create_case()
        self.child, self.parent = self.factory.create_or_update_case(
            CaseStructure(
                relationships=[
                    CaseRelationship(),
                ]
            )
        )
        self.assert_owner_clean()  # this is an actual assertion

    def _verify_set_cleanliness_flags(self):
        """
        Can be run at the end of any relevant test to check the current state of the
        OwnershipCleanliness object and verify that rebuilding it from scratch produces
        the same result
        """
        is_clean = self.owner_cleanliness.is_clean
        hint = self.owner_cleanliness.hint
        self.owner_cleanliness.delete()
        set_cleanliness_flags(self.domain, self.owner_id)
        new_cleanliness = OwnershipCleanlinessFlag.objects.get(owner_id=self.owner_id)
        self.assertEqual(is_clean, new_cleanliness.is_clean)
        self.assertEqual(hint, new_cleanliness.hint)
        if hint:
            self.assertTrue(hint_still_valid(self.domain, self.owner_id, hint))

    @property
    def owner_cleanliness(self):
        return OwnershipCleanlinessFlag.objects.get_or_create(
            owner_id=self.owner_id,
            domain=self.domain,
            defaults={'is_clean': True}
        )[0]

    def assert_owner_clean(self):
        return self.assertTrue(self.owner_cleanliness.is_clean)

    def assert_owner_dirty(self):
        return self.assertFalse(self.owner_cleanliness.is_clean)

    def _set_owner(self, case_id, owner_id):
        case = self.factory.create_or_update_case(
            CaseStructure(case_id=case_id, attrs={'create': False, 'owner_id': owner_id})
        )[0]
        self.assertEqual(owner_id, case.owner_id)

    def test_add_normal_case_stays_clean(self):
        # change the owner ID of a normal case, should remain clean
        self.factory.create_case()
        self.assert_owner_clean()
        self._verify_set_cleanliness_flags()

    def test_change_owner_stays_clean(self):
        # change the owner ID of a normal case, should remain clean
        new_owner = uuid.uuid4().hex
        self._set_owner(self.sample_case._id, new_owner)
        self.assert_owner_clean()
        self._verify_set_cleanliness_flags()

    def test_change_owner_child_case_stays_clean(self):
        # change the owner ID of a child case, should remain clean
        new_owner = uuid.uuid4().hex
        self._set_owner(self.child._id, new_owner)
        self.assert_owner_clean()
        self._verify_set_cleanliness_flags()

    def test_add_clean_parent_stays_clean(self):
        # add a parent with the same owner, should remain clean
        self.factory.create_or_update_case(CaseStructure(relationships=[CaseRelationship()]))
        self.assert_owner_clean()
        self._verify_set_cleanliness_flags()

    def test_create_dirty_makes_dirty(self):
        # create a case and a parent case with a different owner at the same time
        # make sure the owner becomes dirty.
        new_owner = uuid.uuid4().hex
        [child, parent] = self.factory.create_or_update_case(
            CaseStructure(
                relationships=[
                    CaseRelationship(CaseStructure(attrs={'owner_id': new_owner}))
                ]
            )
        )
        self.assert_owner_dirty()
        self.assertEqual(child._id, self.owner_cleanliness.hint)
        self._verify_set_cleanliness_flags()

    def test_add_dirty_parent_makes_dirty(self):
        # add parent with a different owner and make sure the owner becomes dirty
        new_owner = uuid.uuid4().hex
        [child, parent] = self.factory.create_or_update_case(
            CaseStructure(
                case_id=self.sample_case._id,
                relationships=[
                    CaseRelationship(CaseStructure(attrs={'owner_id': new_owner}))
                ]
            )
        )
        self.assert_owner_dirty()
        self.assertEqual(child._id, self.owner_cleanliness.hint)
        self._verify_set_cleanliness_flags()

    def test_change_parent_owner_makes_dirty(self):
        # change the owner id of a parent case and make sure the owner becomes dirty
        new_owner = uuid.uuid4().hex
        self._set_owner(self.parent._id, new_owner)
        self.assert_owner_dirty()
        self.assertEqual(self.child._id, self.owner_cleanliness.hint)
        self._verify_set_cleanliness_flags()

    @override_settings(TESTS_SHOULD_TRACK_CLEANLINESS=None)
    def test_toggle(self):
        # make sure the flag gets removed
        OWNERSHIP_CLEANLINESS.randomness = 0
        # and any test that normally expects a flag to be set to fail
        with self.assertRaises(AssertionError):
            self.test_create_dirty_makes_dirty()

    def test_set_flag_clean_no_data(self):
        unused_owner_id = uuid.uuid4().hex
        set_cleanliness_flags(self.domain, unused_owner_id)
        self.assertTrue(OwnershipCleanlinessFlag.objects.get(owner_id=unused_owner_id).is_clean)

    def test_hint_invalidation(self):
        self.test_change_parent_owner_makes_dirty()
        self._set_owner(self.parent._id, self.owner_id)
        # after the submission the dirtiness flag should still be set
        # since it isn't invalidated right away
        self.assert_owner_dirty()
        # explicitly make sure the hint is no longer valid
        self.assertFalse(hint_still_valid(self.domain, self.owner_id, self.owner_cleanliness.hint))
        # reset the cleanliness flag and ensure it worked
        set_cleanliness_flags(self.domain, self.owner_id)
        self.assert_owner_clean()
        self.assertEqual(None, self.owner_cleanliness.hint)

    def test_cross_domain_on_submission(self):
        # create a form that makes a dirty owner with the same ID but in a different domain
        # make sure the original owner stays clean
        new_domain = uuid.uuid4().hex
        # initialize the new cleanliness flag
        OwnershipCleanlinessFlag.objects.create(domain=new_domain, owner_id=self.owner_id, is_clean=True)
        self.factory.domain = new_domain
        self.factory.create_or_update_case(
            CaseStructure(
                relationships=[
                    CaseRelationship(CaseStructure(attrs={'owner_id': uuid.uuid4().hex}))
                ]
            )
        )
        self.assert_owner_clean()
        self.assertEqual(
            False,
            OwnershipCleanlinessFlag.objects.get(owner_id=self.owner_id, domain=new_domain).is_clean,
        )

    def test_cross_domain_both_clean(self):
        new_domain = uuid.uuid4().hex
        self.factory.domain = new_domain
        self.factory.create_or_update_case(
            CaseStructure(
                relationships=[
                    CaseRelationship(),
                ]
            )
        )
        # two clean ownership models in different domains should report clean
        self.assertTrue(get_cleanliness_flag_from_scratch(self.domain, self.owner_id).is_clean)
        self.assertTrue(get_cleanliness_flag_from_scratch(new_domain, self.owner_id).is_clean)

    def test_cross_domain_dirty(self):
        new_domain = uuid.uuid4().hex
        new_owner = uuid.uuid4().hex
        self.factory.domain = new_domain
        self.factory.create_or_update_case(
            CaseStructure(
                relationships=[
                    CaseRelationship(CaseStructure(attrs={'owner_id': new_owner})),
                ]
            )
        )
        # original domain should stay clean but the new one should be dirty
        self.assertTrue(get_cleanliness_flag_from_scratch(self.domain, self.owner_id).is_clean)
        self.assertFalse(get_cleanliness_flag_from_scratch(new_domain, self.owner_id).is_clean)

    @override_settings(TESTS_SHOULD_TRACK_CLEANLINESS=False)
    def test_autocreate_flag_off(self):
        new_owner = uuid.uuid4().hex
        self.factory.create_or_update_case(
            CaseStructure(case_id=uuid.uuid4().hex, attrs={'create': True, 'owner_id': new_owner})
        )[0]
        self.assertFalse(OwnershipCleanlinessFlag.objects.filter(domain=self.domain, owner_id=new_owner).exists())

    @override_settings(TESTS_SHOULD_TRACK_CLEANLINESS=True)
    def test_autocreate_flag_on(self):
        new_owner = uuid.uuid4().hex
        self.factory.create_or_update_case(
            CaseStructure(case_id=uuid.uuid4().hex, attrs={'create': True, 'owner_id': new_owner})
        )[0]
        flag = OwnershipCleanlinessFlag.objects.get(domain=self.domain, owner_id=new_owner)
        self.assertEqual(True, flag.is_clean)

class CleanlinessUtilitiesTest(SimpleTestCase):

    def test_pop_ids(self):
        five = set(range(5))
        three = pop_ids(five, 3)
        self.assertEqual(3, len(three))
        self.assertEqual(2, len(five))
        self.assertEqual(five | set(three), set(range(5)))

    def test_pop_ids_too_many(self):
        five = set(range(5))
        back = pop_ids(five, 6)
        self.assertEqual(5, len(back))
        self.assertEqual(0, len(five))
        self.assertEqual(set(back), set(range(5)))
<commit_msg>more aggressive override of default cleanliness tracking setting<commit_after>import uuid
from django.test import SimpleTestCase
from django.test.utils import override_settings
from casexml.apps.case.mock import CaseFactory, CaseStructure, CaseRelationship
from casexml.apps.phone.cleanliness import set_cleanliness_flags, hint_still_valid, \
    get_cleanliness_flag_from_scratch
from casexml.apps.phone.data_providers.case.clean_owners import pop_ids
from casexml.apps.phone.models import OwnershipCleanlinessFlag
from casexml.apps.phone.tests.test_sync_mode import SyncBaseTest
from corehq.toggles import OWNERSHIP_CLEANLINESS

class OwnerCleanlinessTest(SyncBaseTest):

    def setUp(self):
        super(OwnerCleanlinessTest, self).setUp()
        # ensure that randomization is on
        OWNERSHIP_CLEANLINESS.randomness = 1
        self.owner_id = uuid.uuid4().hex
        self.synclog_id = uuid.uuid4().hex
        self.domain = uuid.uuid4().hex
        self.factory = CaseFactory(
            domain=self.domain,
            case_defaults={
                'create': True,
                'owner_id': self.owner_id,
                'user_id': self.owner_id,
            }
        )
        self.assert_owner_clean()  # this first call creates the OwnershipCleanliness doc
        self.sample_case = self.factory.create_case()
        self.child, self.parent = self.factory.create_or_update_case(
            CaseStructure(
                relationships=[
                    CaseRelationship(),
                ]
            )
        )
        self.assert_owner_clean()  # this is an actual assertion

    def _verify_set_cleanliness_flags(self):
        """
        Can be run at the end of any relevant test to check the current state of the
        OwnershipCleanliness object and verify that rebuilding it from scratch produces
        the same result
        """
        is_clean = self.owner_cleanliness.is_clean
        hint = self.owner_cleanliness.hint
        self.owner_cleanliness.delete()
        set_cleanliness_flags(self.domain, self.owner_id)
        new_cleanliness = OwnershipCleanlinessFlag.objects.get(owner_id=self.owner_id)
        self.assertEqual(is_clean, new_cleanliness.is_clean)
        self.assertEqual(hint, new_cleanliness.hint)
        if hint:
            self.assertTrue(hint_still_valid(self.domain, self.owner_id, hint))

    @property
    def owner_cleanliness(self):
        return OwnershipCleanlinessFlag.objects.get_or_create(
            owner_id=self.owner_id,
            domain=self.domain,
            defaults={'is_clean': True}
        )[0]

    def assert_owner_clean(self):
        return self.assertTrue(self.owner_cleanliness.is_clean)

    def assert_owner_dirty(self):
        return self.assertFalse(self.owner_cleanliness.is_clean)

    def _set_owner(self, case_id, owner_id):
        case = self.factory.create_or_update_case(
            CaseStructure(case_id=case_id, attrs={'create': False, 'owner_id': owner_id})
        )[0]
        self.assertEqual(owner_id, case.owner_id)

    def test_add_normal_case_stays_clean(self):
        # change the owner ID of a normal case, should remain clean
        self.factory.create_case()
        self.assert_owner_clean()
        self._verify_set_cleanliness_flags()

    def test_change_owner_stays_clean(self):
        # change the owner ID of a normal case, should remain clean
        new_owner = uuid.uuid4().hex
        self._set_owner(self.sample_case._id, new_owner)
        self.assert_owner_clean()
        self._verify_set_cleanliness_flags()

    def test_change_owner_child_case_stays_clean(self):
        # change the owner ID of a child case, should remain clean
        new_owner = uuid.uuid4().hex
        self._set_owner(self.child._id, new_owner)
        self.assert_owner_clean()
        self._verify_set_cleanliness_flags()

    def test_add_clean_parent_stays_clean(self):
        # add a parent with the same owner, should remain clean
        self.factory.create_or_update_case(CaseStructure(relationships=[CaseRelationship()]))
        self.assert_owner_clean()
        self._verify_set_cleanliness_flags()

    def test_create_dirty_makes_dirty(self):
        # create a case and a parent case with a different owner at the same time
        # make sure the owner becomes dirty.
        new_owner = uuid.uuid4().hex
        [child, parent] = self.factory.create_or_update_case(
            CaseStructure(
                relationships=[
                    CaseRelationship(CaseStructure(attrs={'owner_id': new_owner}))
                ]
            )
        )
        self.assert_owner_dirty()
        self.assertEqual(child._id, self.owner_cleanliness.hint)
        self._verify_set_cleanliness_flags()

    def test_add_dirty_parent_makes_dirty(self):
        # add parent with a different owner and make sure the owner becomes dirty
        new_owner = uuid.uuid4().hex
        [child, parent] = self.factory.create_or_update_case(
            CaseStructure(
                case_id=self.sample_case._id,
                relationships=[
                    CaseRelationship(CaseStructure(attrs={'owner_id': new_owner}))
                ]
            )
        )
        self.assert_owner_dirty()
        self.assertEqual(child._id, self.owner_cleanliness.hint)
        self._verify_set_cleanliness_flags()

    def test_change_parent_owner_makes_dirty(self):
        # change the owner id of a parent case and make sure the owner becomes dirty
        new_owner = uuid.uuid4().hex
        self._set_owner(self.parent._id, new_owner)
        self.assert_owner_dirty()
        self.assertEqual(self.child._id, self.owner_cleanliness.hint)
        self._verify_set_cleanliness_flags()

    def test_toggle(self):
        # make sure the flag gets removed
        OWNERSHIP_CLEANLINESS.randomness = 0
        # and any test that normally expects a flag to be set to fail
        with self.assertRaises(AssertionError):
            self.test_create_dirty_makes_dirty()

    def test_set_flag_clean_no_data(self):
        unused_owner_id = uuid.uuid4().hex
        set_cleanliness_flags(self.domain, unused_owner_id)
        self.assertTrue(OwnershipCleanlinessFlag.objects.get(owner_id=unused_owner_id).is_clean)

    def test_hint_invalidation(self):
        self.test_change_parent_owner_makes_dirty()
        self._set_owner(self.parent._id, self.owner_id)
        # after the submission the dirtiness flag should still be set
        # since it isn't invalidated right away
        self.assert_owner_dirty()
        # explicitly make sure the hint is no longer valid
        self.assertFalse(hint_still_valid(self.domain, self.owner_id, self.owner_cleanliness.hint))
        # reset the cleanliness flag and ensure it worked
        set_cleanliness_flags(self.domain, self.owner_id)
        self.assert_owner_clean()
        self.assertEqual(None, self.owner_cleanliness.hint)

    def test_cross_domain_on_submission(self):
        # create a form that makes a dirty owner with the same ID but in a different domain
        # make sure the original owner stays clean
        new_domain = uuid.uuid4().hex
        # initialize the new cleanliness flag
        OwnershipCleanlinessFlag.objects.create(domain=new_domain, owner_id=self.owner_id, is_clean=True)
        self.factory.domain = new_domain
        self.factory.create_or_update_case(
            CaseStructure(
                relationships=[
                    CaseRelationship(CaseStructure(attrs={'owner_id': uuid.uuid4().hex}))
                ]
            )
        )
        self.assert_owner_clean()
        self.assertEqual(
            False,
            OwnershipCleanlinessFlag.objects.get(owner_id=self.owner_id, domain=new_domain).is_clean,
        )

    def test_cross_domain_both_clean(self):
        new_domain = uuid.uuid4().hex
        self.factory.domain = new_domain
        self.factory.create_or_update_case(
            CaseStructure(
                relationships=[
                    CaseRelationship(),
                ]
            )
        )
        # two clean ownership models in different domains should report clean
        self.assertTrue(get_cleanliness_flag_from_scratch(self.domain, self.owner_id).is_clean)
        self.assertTrue(get_cleanliness_flag_from_scratch(new_domain, self.owner_id).is_clean)

    def test_cross_domain_dirty(self):
        new_domain = uuid.uuid4().hex
        new_owner = uuid.uuid4().hex
        self.factory.domain = new_domain
        self.factory.create_or_update_case(
            CaseStructure(
                relationships=[
                    CaseRelationship(CaseStructure(attrs={'owner_id': new_owner})),
                ]
            )
        )
        # original domain should stay clean but the new one should be dirty
        self.assertTrue(get_cleanliness_flag_from_scratch(self.domain, self.owner_id).is_clean)
        self.assertFalse(get_cleanliness_flag_from_scratch(new_domain, self.owner_id).is_clean)

    @override_settings(TESTS_SHOULD_TRACK_CLEANLINESS=False)
    def test_autocreate_flag_off(self):
        new_owner = uuid.uuid4().hex
        self.factory.create_or_update_case(
            CaseStructure(case_id=uuid.uuid4().hex, attrs={'create': True, 'owner_id': new_owner})
        )[0]
        self.assertFalse(OwnershipCleanlinessFlag.objects.filter(domain=self.domain, owner_id=new_owner).exists())

    @override_settings(TESTS_SHOULD_TRACK_CLEANLINESS=True)
    def test_autocreate_flag_on(self):
        new_owner = uuid.uuid4().hex
        self.factory.create_or_update_case(
            CaseStructure(case_id=uuid.uuid4().hex, attrs={'create': True, 'owner_id': new_owner})
        )[0]
        flag = OwnershipCleanlinessFlag.objects.get(domain=self.domain, owner_id=new_owner)
        self.assertEqual(True, flag.is_clean)

OwnerCleanlinessTest = override_settings(TESTS_SHOULD_TRACK_CLEANLINESS=None)(OwnerCleanlinessTest)

class CleanlinessUtilitiesTest(SimpleTestCase):

    def test_pop_ids(self):
        five = set(range(5))
        three = pop_ids(five, 3)
        self.assertEqual(3, len(three))
        self.assertEqual(2, len(five))
        self.assertEqual(five | set(three), set(range(5)))

    def test_pop_ids_too_many(self):
        five = set(range(5))
        back = pop_ids(five, 6)
        self.assertEqual(5, len(back))
        self.assertEqual(0, len(five))
        self.assertEqual(set(back), set(range(5)))
<|endoftext|>


<commit_before>import numpy as np
import scipy.special as sp
import scipy.stats as ss

def draw_data(Rc, Rb, muz, sigmaz, A, mu_noise = np.log(0.01), sigma_noise = 0.1, zmax=0.1):
    Nc = np.random.poisson(Rc)
    Nb = np.random.poisson(Rb)

    czs = muz + sigmaz*np.random.randn(Nc)
    cdzs = np.random.lognormal(mean=mu_noise, sigma=sigma_noise, size=Nc)

    us = np.random.uniform(size=Nb)
    bzs = zmax/(2.0*A-1.0)*(A - np.sqrt(A*A - 2.0*A*us + us))
    bdzs = np.random.lognormal(mean=mu_noise, sigma=sigma_noise, size=Nb)

    zs = np.concatenate((czs, bzs), axis=0)
    dzs = np.concatenate((cdzs, bdzs), axis=0)

    data = np.column_stack((zs, dzs))
    data = np.random.permutation(data)
    data[:,0] += data[:,1]*np.random.randn(data.shape[0])

    return data

class Posterior(object):
    def __init__(self, zs, dzs, zmax=0.1):
        self._zs = zs
        self._dzs = dzs
        self._zmax = zmax

    @property
    def zs(self):
        return self._zs

    @property
    def dzs(self):
        return self._dzs

    @property
    def zmax(self):
        return self._zmax

    @property
    def dtype(self):
        return np.dtype([('Rc', np.float),
                         ('Rb', np.float),
                         ('muz', np.float),
                         ('sigmaz', np.float),
                         ('A', np.float)])

    def to_params(self, p):
        return p.view(self.dtype)

    def log_convolved_foreground_density(self, mu, sigma):
        sigmas = np.sqrt(sigma*sigma + self.dzs*self.dzs)
        return ss.norm.logpdf(self.zs, loc=mu, scale=sigmas)

    def log_convolved_background_density(self, A):
        zmax = self.zmax
        zmax2 = zmax*zmax

        zs = self.zs
        dzs = self.dzs

        term1 = (A*zmax - 2.0*A*zs + zs)*(sp.erf((zmax - zs)/(np.sqrt(2.0)*dzs)) + sp.erf(zs/(np.sqrt(2.0)*dzs)))/zmax2

        term2 = np.sqrt(2.0)*dzs*(2.0*A-1.0)/(np.sqrt(np.pi)*zmax2)*(np.exp(-0.5*np.square(zmax-zs)/np.square(dzs)) - np.exp(-0.5*np.square(zs)/np.square(dzs)))

        return np.log(term1 + term2)

    def log_prior(self, p):
        p = self.to_params(p)

        Rc = p['Rc']
        Rb = p['Rb']
        muz = p['muz']
        sigmaz = p['sigmaz']
        A = p['A']

        if Rc < 0 or Rb < 0:
            return np.NINF

        if sigmaz < 0:
            return np.NINF

        if A < 0 or A > 1:
            return np.NINF

        if muz - 2.0*sigmaz < 0 or muz + 2.0*sigmaz > self.zmax:
            return np.NINF

        if np.abs(A-0.5) > 1e-2:
            A2m1 = 2.0*A - 1.0
            Aprior2 = -(2.0*(4.0*A - np.log(A) + np.log1p(-A) - 2.0))/(A2m1*A2m1*A2m1)
            Aprior = np.sqrt(Aprior2)
        else:
            Aprior = 2.0*np.sqrt(3.0)/3.0 + 4.0/5.0*np.sqrt(3.0)*np.square(A-0.5)

        eps = np.mean(self.dzs)
        sigprior2 = sigmaz / (eps*eps + sigmaz*sigmaz)

        return -0.5*(np.log(Rc) + np.log(Rb)) + np.log(sigprior2) + np.log(Aprior)

    def log_likelihood(self, p):
        p = self.to_params(p)

        Rc = p['Rc']
        Rb = p['Rb']
        muz = p['muz']
        sigmaz = p['sigmaz']
        A = p['A']

        log_pfore = self.log_convolved_foreground_density(muz, sigmaz)
        log_pback = self.log_convolved_background_density(A)

        log_likes = np.logaddexp(np.log(Rc) + log_pfore,
                                 np.log(Rb) + log_pback)

        return np.sum(log_likes) - Rc - Rb

    def __call__(self, p):
        lp = self.log_prior(p)

        if lp == np.NINF:
            return np.NINF
        else:
            ll = self.log_likelihood(p)
            if np.isnan(ll):
                print p
                raise ValueError("ll = NaN")
            return lp + ll
<commit_msg>Added comments (and math!) to posterior.py<commit_after>import numpy as np
import scipy.special as sp
import scipy.stats as ss

def draw_data(Rc, Rb, muz, sigmaz, A, mu_noise = np.log(0.01), sigma_noise = 0.1, zmax=0.1):
    r"""Draws data according to the model.

    The model used is the following:

    For the cluster galaxies,

    .. math::

      z_\mathrm{true} \sim N\left[ \mu_z, \sigma_z \right]

    while for the background galaxies,

    .. math::

      z_\mathrm{true} \sim 2 \left( \frac{A}{z_\mathrm{max}^2} \left(z_\mathrm{max} - z \right) + \frac{1-A}{z_\mathrm{max}^2} z \right)

    (This is just a normalised, linear density profile, with
    :math:`p(z=0) = A/z_mathrm{max}`.)  For all galixes, the observed
    redshift includes a Gaussian error

    .. math::

      p\left( z_\mathrm{obs} | z_\mathrm{true} \right) = N\left[ 0, \sigma_n \right] \left( z_\mathrm{obs} - z_\mathrm{true} \right)

    :param Rc: The Poisson mean number of cluster galaxies.

    :param Rb: The Poisson mean number of background galaxies.

    :param muz: The mean true redshift of the cluster galaxies.

    :param sigmaz: The standard deviation of the cluster galaxy
      redshifts.

    :param A: The probability density at redshift 0 of the background
      galaxies.

    :param mu_noise: The mean parameter of the log-normal distribution
      from which the noise sigmas are drawn.

    :param sigma_noise: The sigma parameter of the log-normal
      distribution from which the noise sigmas are drawn.

    :param zmax: The maximum redshift

    """
    Nc = np.random.poisson(Rc)
    Nb = np.random.poisson(Rb)

    czs = muz + sigmaz*np.random.randn(Nc)
    cdzs = np.random.lognormal(mean=mu_noise, sigma=sigma_noise, size=Nc)

    us = np.random.uniform(size=Nb)
    bzs = zmax/(2.0*A-1.0)*(A - np.sqrt(A*A - 2.0*A*us + us))
    bdzs = np.random.lognormal(mean=mu_noise, sigma=sigma_noise, size=Nb)

    zs = np.concatenate((czs, bzs), axis=0)
    dzs = np.concatenate((cdzs, bdzs), axis=0)

    data = np.column_stack((zs, dzs))
    data = np.random.permutation(data)
    data[:,0] += data[:,1]*np.random.randn(data.shape[0])

    return data

class Posterior(object):
    """Callable object representing the posterior.

    """

    def __init__(self, zs, dzs, zmax=0.1):
        """Initialize the posterior with the given redshifts and
        uncertainties.

        """
        self._zs = zs
        self._dzs = dzs
        self._zmax = zmax

    @property
    def zs(self):
        """The galaxy redshifts."""
        return self._zs

    @property
    def dzs(self):
        """Galaxy redshfit uncertainty."""
        return self._dzs

    @property
    def zmax(self):
        """The maximum true redshift of a galaxy."""
        return self._zmax

    @property
    def dtype(self):
        """Data type describing the parameters of the model."""
        return np.dtype([('Rc', np.float),
                         ('Rb', np.float),
                         ('muz', np.float),
                         ('sigmaz', np.float),
                         ('A', np.float)])

    def to_params(self, p):
        """Returns a view of ``p`` in the parameter data type."""
        return p.view(self.dtype)

    def log_convolved_foreground_density(self, mu, sigma):
        """The log of the foreground density convolved with the observational
        errors.
        """
        sigmas = np.sqrt(sigma*sigma + self.dzs*self.dzs)
        return ss.norm.logpdf(self.zs, loc=mu, scale=sigmas)

    def log_convolved_background_density(self, A):
        """The log of the background density convolved with the observational
        errors.

        """
        zmax = self.zmax
        zmax2 = zmax*zmax

        zs = self.zs
        dzs = self.dzs

        term1 = (A*zmax - 2.0*A*zs + zs)*(sp.erf((zmax - zs)/(np.sqrt(2.0)*dzs)) + sp.erf(zs/(np.sqrt(2.0)*dzs)))/zmax2

        term2 = np.sqrt(2.0)*dzs*(2.0*A-1.0)/(np.sqrt(np.pi)*zmax2)*(np.exp(-0.5*np.square(zmax-zs)/np.square(dzs)) - np.exp(-0.5*np.square(zs)/np.square(dzs)))

        return np.log(term1 + term2)

    def log_prior(self, p):
        r"""The log of the prior.  We use quasi-Jeffreys priors on all
        parameters:

        .. math::

          p(R) \sim \frac{1}{\sqrt{R}}

        .. math::

          p\left(\sigma_z\right) \sim \frac{\sigma_z}{\epsilon^2 + \sigma_z^2}

        where :math:`\epsilon` is the average observational errorbar.

        .. math::

          p\left(\mu_z\right) \sim \mathrm{const}

        .. math::

          p(A) \sim \frac{2A - 1 - \ln A + \ln (1-A)}{\left(1 - 2A\right)^3}

        """
        p = self.to_params(p)

        Rc = p['Rc']
        Rb = p['Rb']
        muz = p['muz']
        sigmaz = p['sigmaz']
        A = p['A']

        if Rc < 0 or Rb < 0:
            return np.NINF

        if sigmaz < 0:
            return np.NINF

        if A < 0 or A > 1:
            return np.NINF

        if muz - 2.0*sigmaz < 0 or muz + 2.0*sigmaz > self.zmax:
            return np.NINF

        if np.abs(A-0.5) > 1e-2:
            A2m1 = 2.0*A - 1.0
            Aprior2 = -(2.0*(4.0*A - np.log(A) + np.log1p(-A) - 2.0))/(A2m1*A2m1*A2m1)
            Aprior = np.sqrt(Aprior2)
        else:
            Aprior = 2.0*np.sqrt(3.0)/3.0 + 4.0/5.0*np.sqrt(3.0)*np.square(A-0.5)

        eps = np.mean(self.dzs)
        sigprior2 = sigmaz / (eps*eps + sigmaz*sigmaz)

        return -0.5*(np.log(Rc) + np.log(Rb)) + np.log(sigprior2) + np.log(Aprior)

    def log_likelihood(self, p):
        """The log-likelihood of our model.

        """
        p = self.to_params(p)

        Rc = p['Rc']
        Rb = p['Rb']
        muz = p['muz']
        sigmaz = p['sigmaz']
        A = p['A']

        log_pfore = self.log_convolved_foreground_density(muz, sigmaz)
        log_pback = self.log_convolved_background_density(A)

        log_likes = np.logaddexp(np.log(Rc) + log_pfore,
                                 np.log(Rb) + log_pback)

        return np.sum(log_likes) - Rc - Rb

    def __call__(self, p):
        """Returns the sum of the log-likelihood and log-prior.

        """
        lp = self.log_prior(p)

        if lp == np.NINF:
            return np.NINF
        else:
            ll = self.log_likelihood(p)
            if np.isnan(ll):
                print p
                raise ValueError("ll = NaN")
            return lp + ll
<|endoftext|>


<commit_before>from setuptools import setup, find_packages

setup(
    name='zeit.content.image',
    version='2.2.7.dev0',
    author='gocept',
    author_email='zon-backend@zeit.de',
    url='https://bitbucket.org/gocept/zeit.content.image',
    description="ZEIT Image",
    packages=find_packages('src'),
    package_dir={'': 'src'},
    include_package_data=True,
    zip_safe=False,
    license='BSD',
    namespace_packages=['zeit', 'zeit.content'],
    install_requires=[
        'ZODB',
        'Pillow',
        'gocept.async',
        'gocept.form',
        'gocept.httpserverlayer',
        'gocept.selenium',
        'grokcore.component',
        'lxml',
        'persistent',
        'pytz',
        'setuptools',
        'transaction',
        'z3c.conditionalviews',
        'zc.form',
        'zc.sourcefactory',
        'zc.table',
        'zeit.cms>=2.31.0.dev0',
        'zeit.connector>=2.4.0.dev0',
        'zeit.edit>=2.1.7.dev0',
        'zeit.imp>=0.15.0.dev0',
        'zeit.wysiwyg',
        'zope.app.appsetup',
        'zope.app.container',
        'zope.app.file',
        'zope.app.generations',
        'zope.app.pagetemplate',
        'zope.browserpage',
        'zope.cachedescriptors',
        'zope.component',
        'zope.file',
        'zope.formlib',
        'zope.interface',
        'zope.publisher',
        'zope.schema',
        'zope.security',
        'zope.testbrowser',
        'zope.testing',
    ],
    entry_points={
        'fanstatic.libraries': [
            'zeit_content_image=zeit.content.image.browser.resources:lib',
        ],
    },
)
<commit_msg>Update copyright<commit_after>from setuptools import setup, find_packages

setup(
    name='zeit.content.image',
    version='2.2.7.dev0',
    author='gocept, Zeit Online',
    author_email='zon-backend@zeit.de',
    url='http://www.zeit.de/',
    description="vivi Content-Type Image, ImageGroup",
    packages=find_packages('src'),
    package_dir={'': 'src'},
    include_package_data=True,
    zip_safe=False,
    license='BSD',
    namespace_packages=['zeit', 'zeit.content'],
    install_requires=[
        'ZODB',
        'Pillow',
        'gocept.async',
        'gocept.form',
        'gocept.httpserverlayer',
        'gocept.selenium',
        'grokcore.component',
        'lxml',
        'persistent',
        'pytz',
        'setuptools',
        'transaction',
        'z3c.conditionalviews',
        'zc.form',
        'zc.sourcefactory',
        'zc.table',
        'zeit.cms>=2.31.0.dev0',
        'zeit.connector>=2.4.0.dev0',
        'zeit.edit>=2.1.7.dev0',
        'zeit.imp>=0.15.0.dev0',
        'zeit.wysiwyg',
        'zope.app.appsetup',
        'zope.app.container',
        'zope.app.file',
        'zope.app.generations',
        'zope.app.pagetemplate',
        'zope.browserpage',
        'zope.cachedescriptors',
        'zope.component',
        'zope.file',
        'zope.formlib',
        'zope.interface',
        'zope.publisher',
        'zope.schema',
        'zope.security',
        'zope.testbrowser',
        'zope.testing',
    ],
    entry_points={
        'fanstatic.libraries': [
            'zeit_content_image=zeit.content.image.browser.resources:lib',
        ],
    },
)
<|endoftext|>


<commit_before># encoding: utf8

import config
import sys
import argparse
import sqlite3
import random
import time
import re
from datetime import datetime
from math import floor
import httplib
import mechanize
from BeautifulSoup import BeautifulSoup
import urllib2

def dict_factory(cursor, row):
    """For convenient dict access of sqlite results"""
    d = {}
    for idx, col in enumerate(cursor.description):
        d[col[0]] = row[idx]
    return d

def init():
    """
    Initialize the database
    """
    cursor = db.cursor()
    cursor.execute("""CREATE TABLE
        IF NOT EXISTS
        jobs
        (num INT)""")
    cursor.execute("""CREATE UNIQUE INDEX
        IF NOT EXISTS
        jobsnum
        ON jobs(num)""")
    cursor.execute("""CREATE TABLE
        IF NOT EXISTS
        organizations
        (
            state TEXT,
            court TEXT,
            register_type TEXT,
            idnum TEXT,
            name TEXT,
            location TEXT,
            last_seen TEXT DEFAULT CURRENT_DATE
        )""")
    cursor.execute("""CREATE UNIQUE INDEX
        IF NOT EXISTS
        orguniq
        ON organizations(court, register_type, idnum)""")
    db.commit()

def empty_jobqueue():
    """Remove all jobs from the queue"""
    cursor = db.cursor()
    cursor.execute("DELETE FROM jobs WHERE 1")
    db.commit()

def empty_orgstable():
    """Delete all organizations"""
    cursor = db.cursor()
    cursor.execute("DELETE FROM organizations WHERE 1")
    db.commit()

def count_jobs():
    """Returns the number of jobs left"""
    cursor = db.cursor()
    cursor.execute("SELECT count(*) AS num_jobs FROM jobs")
    return cursor.fetchone()["num_jobs"]

def has_jobs():
    """Returns true of job queue has jobs"""
    if count_jobs() > 0:
        return True
    return False

def create_jobs():
    """Fills the job queue with jobs"""
    if args.verbose:
        n = config.MAX_REGISTER_NUMBER - config.MIN_REGISTER_NUMBER
        print("Creating %d jobs..." % n)
    cursor = db.cursor()
    for n in range(config.MIN_REGISTER_NUMBER, config.MAX_REGISTER_NUMBER + 1):
        cursor.execute("INSERT INTO jobs (num) VALUES (?)", (n,))
    db.commit()

def get_job():
    """returns a random job"""
    cursor = db.cursor()
    cursor.execute("SELECT num FROM jobs ORDER BY random() LIMIT 1")
    return cursor.fetchone()["num"]

def remove_job(num):
    """Remove the job with the given number from the queue"""
    cursor = db.cursor()
    cursor.execute("DELETE FROM jobs WHERE num=?", (num, ))
    db.commit()

def wait():
    """Wait for a random time interval"""
    t = random.triangular(config.WAIT_MIN, config.WAIT_MAX)
    time.sleep(t)

def duration_string(sec):
    """Formats a time interval to a readable string"""
    sec = float(int(sec))
    if sec > 60 * 60 * 24:
        return "%.1f days" %  (sec / float(60 * 60 * 24))
    if sec > 60 * 60:
        return "%.1f hours" %  (sec / float(60 * 60))
    if sec > 60:
        return "%.1f minutes" %  (sec / float(60))
    return "%d seconds" %  sec

def clean_spaces(t):
    """Remove multiple spaces from a string"""
    if t is None:
        return t
    return re.sub('\s+', ' ', t)

def random_ua_string():
    """Returns a randomly selected user agent strings"""
    return random.sample(config.USER_AGENTS, 1)[0]

def save_result_items(html):
    """Retrieves content from search result HTML"""
    cursor = db.cursor()
    count = 0
    html_parts = re.split('
\s*', html)
    if len(html_parts):
        html = ("
\n\n" + '

').join(html_parts)
        soup = BeautifulSoup(html)
        items = soup.findAll("table", {"class": "scrapableTable"})
        for table in items:
            state_field = table.find("td", {"class": "RegPortErg_AZ"}).contents[0].strip()
            title_field = clean_spaces(table.find("td", {"class": "RegPortErg_AZ"}).find("b").string.strip())
            name_field = table.find("td", {"class": "RegPortErg_FirmaKopf"}).contents[0].strip().replace('&', '&').replace('"', '"').replace(''', "'")
            try:
                location_field = table.find("td", {"class": "RegPortErg_SitzStatusKopf"}).contents[0].strip()
            except IndexError:
                print table.find("td", {"class": "RegPortErg_SitzStatusKopf"})
                location_field = ''

            m = re.search(r"Amtsgericht\s+(.+)\s+([GHRBVPAn]{2,3})\s+([0-9]+.*)", title_field)
            court = None
            register_type = None
            idnum = None
            if m is not None:
                count += 1
                court = m.group(1).strip()
                register_type = m.group(2).strip()
                idnum = m.group(3).strip()
            else:
                sys.stderr.write("PROBLEM: title_field has no match: %s" % title_field)
            record = {
                'state': clean_spaces(state_field),
                'court': court,
                'register_type': register_type,
                'idnum': idnum,
                'name': clean_spaces(name_field),
                'location': clean_spaces(location_field),
                'last_seen': datetime.utcnow().strftime("%Y-%m-%d")
            }
            sql = """INSERT OR REPLACE INTO organizations
                (state, court, register_type, idnum, name, location, last_seen)
                VALUES (?, ?, ?, ?, ?, ?, ?)"""
            cursor.execute(sql, (
                record['state'],
                record['court'],
                record['register_type'],
                record['idnum'],
                record['name'],
                record['location'],
                record['last_seen']))

    db.commit()
    return count

def get_num_results(html):
    """Returns None if result size too large or Int if size is known."""
    pattern = re.compile(r'Ihre Suche hat ([0-9]+) Treffer ergeben')
    match = pattern.search(html)
    if match is not None:
        return int(match.group(1))
    match = re.search('Trefferanzahl von 200 wurde', html)
    if match is not None:
        return None

def resolve_job(regnum):
    """
    Perform a search on handelsregister.de for one specific number
    and store the result.

    If anything fails, we immediately return, which mens the job
    will be taken care of with another attempt.
    """
    global error_1_count
    br = mechanize.Browser()
    br.addheaders = [("User-agent", random_ua_string())]
    br.set_handle_robots(False)

    wait()
    try:
        br.open(config.HANDELSREGISTER_URL).read()
    except httplib.IncompleteRead:
        if args.verbose:
            sys.stderr.write("Error 4: Incomplete response.\n")
        return
    except urllib2.URLError:
        if args.verbose:
            sys.stderr.write("Error 5: URLError\n")
        return
    except socks.SOCKS5Error, e:
        sys.stderr.write("Fatal error: %s\n" % e)
        sys.exit(2)

    allforms = list(br.forms())
    if len(allforms) == 0:
        error_1_count += 1
        sys.stderr.write("Error 1: No form found on initial page.\n")
        if error_1_count >= config.ERROR_1_LIMIT:
            sys.stderr.write("Maximum of %d error 1 occurrences reached. Exiting.\n" % config.ERROR_1_LIMIT)
            sys.exit(1)
        return

    br.form = allforms[0]
    br['registerNummer'] = str(regnum)
    br['ergebnisseProSeite'] = [str(config.ITEMS_PER_PAGE)]

    wait()
    response = br.submit()
    try:
        html = response.read()
    except httplib.IncompleteRead:
        if args.verbose:
            sys.stderr.write("Error 6: Incomplete response.\n")
        return

    num_results = get_num_results(html)
    if num_results is None:
        sys.stderr.write("Error 7: No results on page.\n")
        return
    if args.verbose:
        if num_results == 0:
            pass
        elif num_results == 1:
            print("1 entry found")
        else:
            print("%d entries found" % num_results)

    # while search result is too big, hit "double result" link
    while num_results is None:
        wait()
        br.click_link(url='/rp_web/search.do?doppelt')
        html = br.response().read()
        num_results = get_num_results(html)

    num_found = save_result_items(html)
    current_page = 1
    if num_results > config.ITEMS_PER_PAGE:
        num_pages = int(floor(num_results / config.ITEMS_PER_PAGE) + 1)
        while current_page < num_pages:
            wait()
            try:
                response = br.follow_link(text=str(current_page + 1))
                html = response.read()
                num_found += save_result_items(html)
            except mechanize._mechanize.LinkNotFoundError:
                if args.verbose:
                    sys.stderr.write("Error 3: Pagination link %d not found.\n" % (current_page + 1))
                return
            current_page += 1
    if num_found != num_results:
        if args.verbose:
            sys.stderr.write("Error 2: Number of retrieved entries (%d) does not match indicated number of hits (%d).\n" % (num_found, num_results))
        return

    remove_job(j)
    return True

if __name__ == "__main__":

    # set up command line options
    argparser = argparse.ArgumentParser()
    argparser.add_argument("-j", "--resetjobs", dest="resetjobs",
        action="store_true", default=False,
        help="Empty and recreate the job queue before starting")
    argparser.add_argument("--resetorgs", dest="resetorgs",
        action="store_true", default=False,
        help="Empty the organization table before starting (data loss!)")
    argparser.add_argument("-s", "--socksproxy", dest="socksproxy", default="",
        help="Use this SOCKS5 proxy server (e. g. for use woth Tor)")
    argparser.add_argument("-p", "--proxyport", dest="proxyport",
        type=int, default=config.DEFAULT_PROXY_PORT,
        help="Use this SOCKS5 proxy server (e. g. for use woth Tor)")
    argparser.add_argument("-v", "--verbose", dest="verbose", action="store_true",
        help="Give verbose output")
    args = argparser.parse_args()

    db = sqlite3.connect(config.DB_PATH)
    db.row_factory = dict_factory

    # set up the database
    init()

    if args.resetorgs:
        empty_orgstable()

    if args.resetjobs:
        empty_jobqueue()

    if not has_jobs():
        create_jobs()

    # set up proxy
    if args.socksproxy != "":
        import socks
        import socket
        socks.set_default_proxy(socks.SOCKS5, args.socksproxy, args.proxyport)
        socket.socket = socks.socksocket

    # counter for certain errors
    error_1_count = 0

    starttime = time.time()
    jobs_done = 0

    # the main loop
    while has_jobs():
        c = count_jobs()
        j = get_job()

        if args.verbose:
            print("Register number: %d" % j)

        if resolve_job(j):
            jobs_done += 1

        # display some progress stats
        if args.verbose:
            if jobs_done > 0 and jobs_done % 10 == 0:
                duration = time.time() - starttime
                seconds_per_job = duration / jobs_done
                print("Resolved %d jobs in %s. %d jobs, %s remaining" % (
                    jobs_done, duration_string(duration), c, duration_string(seconds_per_job * c)))

    db.close()
<commit_msg>Catch one more exception<commit_after># encoding: utf8

import config
import sys
import argparse
import sqlite3
import random
import time
import re
from datetime import datetime
from math import floor
import httplib
import mechanize
from BeautifulSoup import BeautifulSoup
import urllib2

def dict_factory(cursor, row):
    """For convenient dict access of sqlite results"""
    d = {}
    for idx, col in enumerate(cursor.description):
        d[col[0]] = row[idx]
    return d

def init():
    """
    Initialize the database
    """
    cursor = db.cursor()
    cursor.execute("""CREATE TABLE
        IF NOT EXISTS
        jobs
        (num INT)""")
    cursor.execute("""CREATE UNIQUE INDEX
        IF NOT EXISTS
        jobsnum
        ON jobs(num)""")
    cursor.execute("""CREATE TABLE
        IF NOT EXISTS
        organizations
        (
            state TEXT,
            court TEXT,
            register_type TEXT,
            idnum TEXT,
            name TEXT,
            location TEXT,
            last_seen TEXT DEFAULT CURRENT_DATE
        )""")
    cursor.execute("""CREATE UNIQUE INDEX
        IF NOT EXISTS
        orguniq
        ON organizations(court, register_type, idnum)""")
    db.commit()

def empty_jobqueue():
    """Remove all jobs from the queue"""
    cursor = db.cursor()
    cursor.execute("DELETE FROM jobs WHERE 1")
    db.commit()

def empty_orgstable():
    """Delete all organizations"""
    cursor = db.cursor()
    cursor.execute("DELETE FROM organizations WHERE 1")
    db.commit()

def count_jobs():
    """Returns the number of jobs left"""
    cursor = db.cursor()
    cursor.execute("SELECT count(*) AS num_jobs FROM jobs")
    return cursor.fetchone()["num_jobs"]

def has_jobs():
    """Returns true of job queue has jobs"""
    if count_jobs() > 0:
        return True
    return False

def create_jobs():
    """Fills the job queue with jobs"""
    if args.verbose:
        n = config.MAX_REGISTER_NUMBER - config.MIN_REGISTER_NUMBER
        print("Creating %d jobs..." % n)
    cursor = db.cursor()
    for n in range(config.MIN_REGISTER_NUMBER, config.MAX_REGISTER_NUMBER + 1):
        cursor.execute("INSERT INTO jobs (num) VALUES (?)", (n,))
    db.commit()

def get_job():
    """returns a random job"""
    cursor = db.cursor()
    cursor.execute("SELECT num FROM jobs ORDER BY random() LIMIT 1")
    return cursor.fetchone()["num"]

def remove_job(num):
    """Remove the job with the given number from the queue"""
    cursor = db.cursor()
    cursor.execute("DELETE FROM jobs WHERE num=?", (num, ))
    db.commit()

def wait():
    """Wait for a random time interval"""
    t = random.triangular(config.WAIT_MIN, config.WAIT_MAX)
    time.sleep(t)

def duration_string(sec):
    """Formats a time interval to a readable string"""
    sec = float(int(sec))
    if sec > 60 * 60 * 24:
        return "%.1f days" %  (sec / float(60 * 60 * 24))
    if sec > 60 * 60:
        return "%.1f hours" %  (sec / float(60 * 60))
    if sec > 60:
        return "%.1f minutes" %  (sec / float(60))
    return "%d seconds" %  sec

def clean_spaces(t):
    """Remove multiple spaces from a string"""
    if t is None:
        return t
    return re.sub('\s+', ' ', t)

def random_ua_string():
    """Returns a randomly selected user agent strings"""
    return random.sample(config.USER_AGENTS, 1)[0]

def save_result_items(html):
    """Retrieves content from search result HTML"""
    cursor = db.cursor()
    count = 0
    html_parts = re.split('
\s*', html)
    if len(html_parts):
        html = ("
\n\n" + '

').join(html_parts)
        soup = BeautifulSoup(html)
        items = soup.findAll("table", {"class": "scrapableTable"})
        for table in items:
            state_field = table.find("td", {"class": "RegPortErg_AZ"}).contents[0].strip()
            title_field = clean_spaces(table.find("td", {"class": "RegPortErg_AZ"}).find("b").string.strip())
            name_field = table.find("td", {"class": "RegPortErg_FirmaKopf"}).contents[0].strip().replace('&', '&').replace('"', '"').replace(''', "'")
            try:
                location_field = table.find("td", {"class": "RegPortErg_SitzStatusKopf"}).contents[0].strip()
            except IndexError:
                print table.find("td", {"class": "RegPortErg_SitzStatusKopf"})
                location_field = ''

            m = re.search(r"Amtsgericht\s+(.+)\s+([GHRBVPAn]{2,3})\s+([0-9]+.*)", title_field)
            court = None
            register_type = None
            idnum = None
            if m is not None:
                count += 1
                court = m.group(1).strip()
                register_type = m.group(2).strip()
                idnum = m.group(3).strip()
            else:
                sys.stderr.write("PROBLEM: title_field has no match: %s" % title_field)
            record = {
                'state': clean_spaces(state_field),
                'court': court,
                'register_type': register_type,
                'idnum': idnum,
                'name': clean_spaces(name_field),
                'location': clean_spaces(location_field),
                'last_seen': datetime.utcnow().strftime("%Y-%m-%d")
            }
            sql = """INSERT OR REPLACE INTO organizations
                (state, court, register_type, idnum, name, location, last_seen)
                VALUES (?, ?, ?, ?, ?, ?, ?)"""
            cursor.execute(sql, (
                record['state'],
                record['court'],
                record['register_type'],
                record['idnum'],
                record['name'],
                record['location'],
                record['last_seen']))

    db.commit()
    return count

def get_num_results(html):
    """Returns None if result size too large or Int if size is known."""
    pattern = re.compile(r'Ihre Suche hat ([0-9]+) Treffer ergeben')
    match = pattern.search(html)
    if match is not None:
        return int(match.group(1))
    match = re.search('Trefferanzahl von 200 wurde', html)
    if match is not None:
        return None

def resolve_job(regnum):
    """
    Perform a search on handelsregister.de for one specific number
    and store the result.

    If anything fails, we immediately return, which mens the job
    will be taken care of with another attempt.
    """
    global error_1_count
    br = mechanize.Browser()
    br.addheaders = [("User-agent", random_ua_string())]
    br.set_handle_robots(False)

    wait()
    try:
        br.open(config.HANDELSREGISTER_URL).read()
    except httplib.IncompleteRead:
        if args.verbose:
            sys.stderr.write("Error 4: Incomplete response.\n")
        return
    except urllib2.URLError:
        if args.verbose:
            sys.stderr.write("Error 5: URLError\n")
        return
    except socks.SOCKS5Error, e:
        sys.stderr.write("Fatal error: %s\n" % e)
        sys.exit(2)

    allforms = list(br.forms())
    if len(allforms) == 0:
        error_1_count += 1
        sys.stderr.write("Error 1: No form found on initial page.\n")
        if error_1_count >= config.ERROR_1_LIMIT:
            sys.stderr.write("Maximum of %d error 1 occurrences reached. Exiting.\n" % config.ERROR_1_LIMIT)
            sys.exit(1)
        return

    br.form = allforms[0]
    br['registerNummer'] = str(regnum)
    br['ergebnisseProSeite'] = [str(config.ITEMS_PER_PAGE)]

    wait()
    try:
        response = br.submit()
    except httplib.BadStatusLine:
        if args.verbose:
            sys.stderr.write("Error 8: Bad HTTP Status.\n")
        return
    try:
        html = response.read()
    except httplib.IncompleteRead:
        if args.verbose:
            sys.stderr.write("Error 6: Incomplete response.\n")
        return

    num_results = get_num_results(html)
    if num_results is None:
        sys.stderr.write("Error 7: No results on page.\n")
        return
    if args.verbose:
        if num_results == 0:
            pass
        elif num_results == 1:
            print("1 entry found")
        else:
            print("%d entries found" % num_results)

    # while search result is too big, hit "double result" link
    while num_results is None:
        wait()
        br.click_link(url='/rp_web/search.do?doppelt')
        html = br.response().read()
        num_results = get_num_results(html)

    num_found = save_result_items(html)
    current_page = 1
    if num_results > config.ITEMS_PER_PAGE:
        num_pages = int(floor(num_results / config.ITEMS_PER_PAGE) + 1)
        while current_page < num_pages:
            wait()
            try:
                response = br.follow_link(text=str(current_page + 1))
                html = response.read()
                num_found += save_result_items(html)
            except mechanize._mechanize.LinkNotFoundError:
                if args.verbose:
                    sys.stderr.write("Error 3: Pagination link %d not found.\n" % (current_page + 1))
                return
            current_page += 1
    if num_found != num_results:
        if args.verbose:
            sys.stderr.write("Error 2: Number of retrieved entries (%d) does not match indicated number of hits (%d).\n" % (num_found, num_results))
        return

    remove_job(j)
    return True

if __name__ == "__main__":

    # set up command line options
    argparser = argparse.ArgumentParser()
    argparser.add_argument("-j", "--resetjobs", dest="resetjobs",
        action="store_true", default=False,
        help="Empty and recreate the job queue before starting")
    argparser.add_argument("--resetorgs", dest="resetorgs",
        action="store_true", default=False,
        help="Empty the organization table before starting (data loss!)")
    argparser.add_argument("-s", "--socksproxy", dest="socksproxy", default="",
        help="Use this SOCKS5 proxy server (e. g. for use woth Tor)")
    argparser.add_argument("-p", "--proxyport", dest="proxyport",
        type=int, default=config.DEFAULT_PROXY_PORT,
        help="Use this SOCKS5 proxy server (e. g. for use woth Tor)")
    argparser.add_argument("-v", "--verbose", dest="verbose", action="store_true",
        help="Give verbose output")
    args = argparser.parse_args()

    db = sqlite3.connect(config.DB_PATH)
    db.row_factory = dict_factory

    # set up the database
    init()

    if args.resetorgs:
        empty_orgstable()

    if args.resetjobs:
        empty_jobqueue()

    if not has_jobs():
        create_jobs()

    # set up proxy
    if args.socksproxy != "":
        import socks
        import socket
        socks.set_default_proxy(socks.SOCKS5, args.socksproxy, args.proxyport)
        socket.socket = socks.socksocket

    # counter for certain errors
    error_1_count = 0

    starttime = time.time()
    jobs_done = 0

    # the main loop
    while has_jobs():
        c = count_jobs()
        j = get_job()

        if args.verbose:
            print("Register number: %d" % j)

        if resolve_job(j):
            jobs_done += 1

        # display some progress stats
        if args.verbose:
            if jobs_done > 0 and jobs_done % 10 == 0:
                duration = time.time() - starttime
                seconds_per_job = duration / jobs_done
                print("Resolved %d jobs in %s. %d jobs, %s remaining" % (
                    jobs_done, duration_string(duration), c, duration_string(seconds_per_job * c)))

    db.close()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""
Created on Tue Jul  5 16:30:53 2016

calibrates using fisheye distortion model (polynomial in theta)

help in
http://docs.opencv.org/ref/master/d9/d0c/group__calib3d.html#ga3207604e4b1a1758aa66acb6ed5aa65d&gsc.tab=0

@author: sebalander
"""

# %%
import cv2
import numpy as np
import glob
import matplotlib.pyplot as plt

# %% LOAD DATA
#imagesFolder = "./resources/fishChessboardImg/"
#cornersFile = "/home/sebalander/code/sebaPhD/resources/fishCorners.npy"
#patternFile = "/home/sebalander/code/sebaPhD/resources/chessPattern.npy"
#imgShapeFile = "/home/sebalander/code/sebaPhD/resources/fishShape.npy"

imagesFolder = "./resources/PTZchessboard/zoom 0.0/"
cornersFile = "/home/sebalander/Code/sebaPhD/resources/PTZchessboard/zoom 0.0/ptzCorners.npy"
patternFile = "/home/sebalander/Code/sebaPhD/resources/chessPattern.npy"
imgShapeFile = "/home/sebalander/Code/sebaPhD/resources/ptzImgShape.npy"

corners = np.load(cornersFile)
fiducialPoints = np.load(patternFile)
imgSize = tuple(np.load(imgShapeFile))
images = glob.glob(imagesFolder+'*.png')

# output files
distCoeffsFile = "/home/sebalander/Code/sebaPhD/resources/PTZchessboard/zoom 0.0/ptzDistCoeffs.npy"
linearCoeffsFile = "/home/sebalander/Code/sebaPhD/resources/PTZchessboard/zoom 0.0/ptzLinearCoeffs.npy"
rvecsFile = "/home/sebalander/Code/sebaPhD/resources/PTZchessboard/zoom 0.0/ptzRvecs.npy"
tvecsFile = "/home/sebalander/Code/sebaPhD/resources/PTZchessboard/zoom 0.0/ptzTvecs.npy"

# %% LINEAR APPROACH TO CAMERA CALIBRATION
n = corners.shape[0] # number of images
m = corners.shape[2] # points per image

#i = 3
#j = 31
#x = corners[i,0,j]
#p = fiducialPoints[0,j]
cer = np.zeros(4)

# los 1s se agregan porque tiene que se en coordenadas homogéneas
def Pelement(x,p):
    a = np.concatenate((p, [1], cer, -x[0]*p, [-x[0]]))
    b = np.concatenate((cer, p, [1], -x[1]*p, [-x[1]]))
    return np.concatenate(([a] ,[b] ),0)

PP = np.array([[Pelement(corners[i,0,j],fiducialPoints[0,j]) for j in range(m)] for i in range(n)])

PP = np.reshape(np.array(PP),(2*n*m,12),order='F')

# check that there are more rows than columns (12)
# make square
PTP = PP.T.dot(PP)
l, v = np.linalg.eig(PTP)

# %% WHAT FOLLOWS IS OBSOLETE
n = len(imgpoints)  # cantidad de imagenes
# Parametros de entrada/salida de la calibracion
rvecs = ()
tvecs = ()
cameraMatrix = ()
distCoeffs = ()

# enable possible coefficients
flags = cv2.CALIB_ZERO_TANGENT_DIST + cv2.CALIB_RATIONAL_MODEL

objpoints = [chessboardModel]*n

calibrationCriteria = (cv2.TERM_CRITERIA_EPS
                       + cv2.TERM_CRITERIA_MAX_ITER,
                       1000, # max number of iterations
                       0.0001) # min accuracy

# %% OPTIMIZAR
rms, cameraMatrix, distCoeffs, rvecs, tvecs = cv2.calibrateCamera(objpoints,
                                                                  imgpoints,
                                                                  imgSize,
                                                                  cameraMatrix,
                                                                  distCoeffs,
                                                                  rvecs,
                                                                  tvecs,
                                                                  flags)

# (k1,k2,p1,p2[,k3[,k4,k5,k6[,s1,s2,s3,s4[,τx,τy]]]])

# %% SAVE PARAMETERS, INTRINSIC AND EXTRINSIC
np.save(distCoeffsFile, distCoeffs)
np.save(linearCoeffsFile, cameraMatrix)
np.save(rvecsFile, rvecs)
np.save(tvecsFile, tvecs)<commit_msg>Scheme of intrinsic calibration layed out<commit_after># -*- coding: utf-8 -*-
"""
Created on Tue Jul  5 16:30:53 2016

calibrates using fisheye distortion model (polynomial in theta)

help in
http://docs.opencv.org/ref/master/d9/d0c/group__calib3d.html#ga3207604e4b1a1758aa66acb6ed5aa65d&gsc.tab=0

@author: sebalander
"""

# %%
import cv2
import numpy as np
import glob
import matplotlib.pyplot as plt

# %% LOAD DATA
#imagesFolder = "./resources/fishChessboardImg/"
#cornersFile = "/home/sebalander/code/sebaPhD/resources/fishCorners.npy"
#patternFile = "/home/sebalander/code/sebaPhD/resources/chessPattern.npy"
#imgShapeFile = "/home/sebalander/code/sebaPhD/resources/fishShape.npy"

imagesFolder = "./resources/PTZchessboard/zoom 0.0/"
cornersFile = "/home/sebalander/Code/sebaPhD/resources/PTZchessboard/zoom 0.0/ptzCorners.npy"
patternFile = "/home/sebalander/Code/sebaPhD/resources/chessPattern.npy"
imgShapeFile = "/home/sebalander/Code/sebaPhD/resources/ptzImgShape.npy"

corners = np.load(cornersFile)
fiducialPoints = np.load(patternFile)
imgSize = tuple(np.load(imgShapeFile))
images = glob.glob(imagesFolder+'*.png')

# output files
distCoeffsFile = "/home/sebalander/Code/sebaPhD/resources/PTZchessboard/zoom 0.0/ptzDistCoeffs.npy"
linearCoeffsFile = "/home/sebalander/Code/sebaPhD/resources/PTZchessboard/zoom 0.0/ptzLinearCoeffs.npy"
rvecsFile = "/home/sebalander/Code/sebaPhD/resources/PTZchessboard/zoom 0.0/ptzRvecs.npy"
tvecsFile = "/home/sebalander/Code/sebaPhD/resources/PTZchessboard/zoom 0.0/ptzTvecs.npy"

# %% LINEAR APPROACH TO CAMERA CALIBRATION
n = corners.shape[0] # number of images
m = corners.shape[2] # points per image

#i = 3
#j = 31
#x = corners[i,0,j]
#p = fiducialPoints[0,j]
cer = np.zeros(4)

def linea1(x,y,xp,yp):
    return [x, 0, -x*xp, y, 0, -y*xp, 1, 0, -xp]

def linea2(x,y,xp,yp):
    return [0, x, -x*yp, 0, y, -y*yp, 0, 1, -yp]

def lineas(x,y,xp,yp):

    [[x, 0, -x*xp, y, 0, -y*xp, 1, 0, -xp],
     [0, x, -x*yp, 0, y, -y*yp, 0, 1, -yp]]

    return np.array([linea1(x,y,xp,yp),linea2(x,y,xp,yp)])

i = 0 # indice de imagen, hasta n
j = 0 # indice de punto, hasta m

x = corners[i,0,j,0]
y = corners[i,0,j,1]
xp = fiducialPoints[0,j,0]
yp = fiducialPoints[0,j,1]

lineas(x,y,xp,yp)

def H(fiducialPoints,corners):
    m = fiducialPoints.shape[1]
    h = [ lineas(fiducialPoints[0,j,0], fiducialPoints[0,j,1],
                 corners[0,j,0], corners[0,j,1])
     for j in range(m)]

    h = np.array(h)
    h = np.reshape(h,(m*2,9))
    return h

h = H(corners[0],fiducialPoints)

h2 = np.matmul(h.T,h)

if np.linalg.matrix_rank(h2)==9:
    "tiene rango 9, signifique lo que signifique"

L, V = np.linalg.eigh(h2)

U, S, V = np.linalg.svd(h)
# elijo el menor
v = V[8]
# chequear que no este repetido?
r1 = v[0:3]
r2 = v[3:6]
t = v[6:9]

np.dot(r1, r2) # no dan perpendiculares
r3 = np.cross(r1,r2)

n1 = np.linalg.norm(r1)
n2 = np.linalg.norm(r2) # las normas dan cualquier cosa
n3 = np.linalg.norm(r3)
nt = np.linalg.norm(t)

# componer matriz de rotacion cruda
# ortonormalizarla

# normalizar los vectores, sacar el de rodrigues

# %%
# hacer optimizacion para terminar de ajustar el pinhole y asi sacar la pose para cada imagen. usar las funciones de las librerias extrinsecas
# es para cada imagen por separado

# %%
# para todas las imagenes juntas dejar la pose fija y optimizar los parametros de distorsion

# %% optimizar todo junto a la vez tomando lo anterior como condiciones iniciales.<|endoftext|>