<commit_before>import logging
import textwrap

from pylons import request
from pylons import tmpl_context as c
from pylons.decorators import validate
from pylons.controllers.util import redirect
from pylons.i18n import _

import formencode
from formencode import validators, htmlfill

from adhocracy import config
from adhocracy import forms
from adhocracy.controllers.instance import InstanceController
from adhocracy.lib.auth import require
from adhocracy.lib.auth.authorization import has
from adhocracy.lib.auth.csrf import RequireInternalRequest
from adhocracy.lib import helpers as h
from adhocracy.lib.message import render_body
from adhocracy.lib.message import send as send_message
from adhocracy.lib.base import BaseController
from adhocracy.lib.templating import render, ret_abort, ret_success
from adhocracy.lib.util import get_entity_or_abort
from adhocracy.lib import democracy
from adhocracy.model import Instance
from adhocracy.model import Membership
from adhocracy.model import Permission
from adhocracy.model import User
from adhocracy.model import UserBadge
from adhocracy.model import UserBadges
from adhocracy.model import Proposal

log = logging.getLogger(__name__)

class MassmessageBaseForm(formencode.Schema):
    allow_extra_fields = True
    subject = validators.String(max=140, not_empty=True)
    body = validators.String(min=2, not_empty=True)

class MassmessageForm(MassmessageBaseForm):
    filter_instances = forms.MessageableInstances(not_empty=True)
    filter_badges = forms.ValidUserBadges()
    sender_email = validators.String(not_empty=True)
    sender_name = validators.String(not_empty=False, if_missing=None)
    include_footer = formencode.validators.StringBoolean(if_missing=False)

class MassmessageProposalForm(MassmessageBaseForm):
    creators = validators.StringBool(not_empty=False, if_empty=False,
                                     if_missing=False)
    supporters = validators.StringBool(not_empty=False, if_empty=False,
                                       if_missing=False)
    opponents = validators.StringBool(not_empty=False, if_empty=False,
                                      if_missing=False)
    chained_validators = [
        forms.ProposalMessageNoRecipientGroup(),
    ]

def _get_options(func):
    """ Decorator that calls the functions with the following parameters:
        sender_email - Email address of the sender
        sender_name  - Name of the sender
        subject      - Subject of the message
        body         - Body of the message
        recipients   - A list of users the email is going to
    """
    @RequireInternalRequest(methods=['POST'])
    @validate(schema=MassmessageForm(), form='new')
    def wrapper(self):
        allowed_sender_options = self._get_allowed_sender_options(c.user)
        sender_email = self.form_result.get('sender_email')
        if ((sender_email not in allowed_sender_options) or
                (not allowed_sender_options[sender_email]['enabled'])):
            return ret_abort(_("Sorry, but you're not allowed to set these "
                               "message options"), code=403)
        sender_name = None
        if has('global.message'):
            sender_name = self.form_result.get('sender_name')
        if not sender_name:
            sender_name = c.user.name

        recipients = User.all_q()
        filter_instances = self.form_result.get('filter_instances')
        recipients = recipients.join(Membership).filter(
            Membership.instance_id.in_(filter_instances))
        filter_badges = self.form_result.get('filter_badges')
        if filter_badges:
            recipients = recipients.join(UserBadges,
                                         UserBadges.user_id == User.id)
            recipients = recipients.filter(
                UserBadges.badge_id.in_([fb.id for fb in filter_badges]))

        if has('global.admin'):
            include_footer = self.form_result.get('include_footer')
        else:
            include_footer = True

        if len(filter_instances) == 1:
            instance = Instance.find(filter_instances[0])
        else:
            instance = None

        return func(self,
                    self.form_result.get('subject'),
                    self.form_result.get('body'),
                    recipients.all(),
                    sender_email=allowed_sender_options[sender_email]['email'],
                    sender_name=sender_name,
                    instance=instance,
                    include_footer=include_footer,
                    )
    return wrapper

class MassmessageController(BaseController):
    """
    This deals with messages to multiple users at the same time. This will be
    will be merged with the MessageController at some point.
    """

    @classmethod
    def _get_allowed_instances(cls, user):
        """
        returns all instances in which the given user has permission to send a
        message to all users
        """
        if has('global.message'):
            return Instance.all()
        else:
            perm = Permission.find('instance.message')
            instances = [m.instance for m in user.memberships
                         if (m.instance is not None
                             and m.instance.is_authenticated
                             and perm in m.group.permissions)]
            return sorted(instances, key=lambda i: i.label)

    @classmethod
    def _get_allowed_sender_options(cls, user):
        sender_options = {
            'user': {
                'email': user.email,
                'checked': False,
                'enabled': user.is_email_activated(),
                'reason': _("Email isn't activated"),
            },
            'system': {
                'email': config.get('adhocracy.email.from'),
                'checked': False,
                'enabled': config.get_bool(
                    'allow_system_email_in_mass_messages'),
                'reason': _("Not permitted in system settings"),
            },
            'support': {
                'email': config.get('adhocracy.registration_support_email'),
                'checked': False,
                'enabled': (config.get('adhocracy.registration_support_email')
                            is not None),
                'reason': _("adhocracy.registration_support_email not set"),
            }
        }

        if sender_options['user']['enabled']:
            sender_options['user']['checked'] = True
        elif sender_options['system']['enabled']:
            sender_options['system']['checked'] = True

        return sender_options

    def new(self, id=None, errors={}, format=u'html'):

        if id is None:
            require.perm('global.message')
            template = '/massmessage/new.html'
            c.preview_url = h.base_url('/message/preview')
        else:
            c.page_instance = InstanceController._get_current_instance(id)
            require.instance.message(c.page_instance)
            template = '/instance/message.html'
            c.preview_url = h.base_url(
                '/instance/%s/message/preview' % id)

        defaults = dict(request.params)
        defaults.setdefault('include_footer', 'on')

        data = {
            'instances': self._get_allowed_instances(c.user),
            'sender_options': self._get_allowed_sender_options(c.user),
            'userbadges': UserBadge.all(instance=c.instance,
                                        include_global=True)
        }

        return htmlfill.render(render(template, data,
                                      overlay=format == u'overlay'),
                               defaults=defaults, errors=errors,
                               force_defaults=False)

    @_get_options
    def preview(self, subject, body, recipients, sender_email, sender_name,
                instance, include_footer):
        recipients_list = sorted(list(recipients), key=lambda r: r.name)
        if recipients_list:
            try:
                rendered_body = render_body(body, recipients_list[0],
                                            is_preview=True)
            except (KeyError, ValueError) as e:
                rendered_body = _('Could not render message: %s') % str(e)
        else:
            rendered_body = body

        # wrap body, but leave long words (e.g. links) intact
        rendered_body = u'\n'.join(textwrap.fill(line, break_long_words=False)
                                   for line in rendered_body.split(u'\n'))

        data = {
            'sender_email': sender_email,
            'sender_name': sender_name,
            'subject': subject,
            'body': rendered_body,
            'recipients': recipients_list,
            'recipients_count': len(recipients_list),
            'params': request.params,
            'include_footer': include_footer,
            'instance': instance,
        }
        return render('/massmessage/preview.html', data)

    @_get_options
    def create(self, subject, body, recipients, sender_email, sender_name,
               instance, include_footer):
        send_message(subject, body, c.user, recipients,
                     sender_email=sender_email,
                     sender_name=sender_name,
                     instance=instance,
                     include_footer=include_footer)
        return ret_success(
            message=_("Message sent to %d users.") % len(recipients))

    def new_proposal(self, proposal_id, errors={}, format=u'html'):
        c.proposal = get_entity_or_abort(Proposal, proposal_id)
        require.proposal.message(c.proposal)
        defaults = dict(request.params)
        return htmlfill.render(render('/massmessage/new_proposal.html',
                                      overlay=format == u'overlay'),
                               defaults=defaults, errors=errors,
                               force_defaults=False)

    @validate(schema=MassmessageProposalForm(), form='new_proposal')
    def create_proposal(self, proposal_id):
        c.proposal = get_entity_or_abort(Proposal, proposal_id)
        require.proposal.message(c.proposal)

        recipients = set()
        if self.form_result.get(u'supporters'):
            recipients.update(democracy.supporters(c.proposal.rate_poll))
        if self.form_result.get(u'opponents'):
            recipients.update(democracy.opponents(c.proposal.rate_poll))
        if self.form_result.get(u'creators'):
            recipients.update(c.proposal.get_creators())

        send_message(self.form_result.get('subject'),
                     self.form_result.get('body'),
                     c.user,
                     recipients,
                     instance=c.instance)
        h.flash(_("Message sent to %d users.") % len(recipients), 'success')
        redirect(h.entity_url(c.proposal))
<commit_msg>Allow global admins to send massmessages to users of hidden instances<commit_after>import logging
import textwrap

from pylons import request
from pylons import tmpl_context as c
from pylons.decorators import validate
from pylons.controllers.util import redirect
from pylons.i18n import _

import formencode
from formencode import validators, htmlfill

from adhocracy import config
from adhocracy import forms
from adhocracy.controllers.instance import InstanceController
from adhocracy.lib.auth import require
from adhocracy.lib.auth.authorization import has
from adhocracy.lib.auth.csrf import RequireInternalRequest
from adhocracy.lib import helpers as h
from adhocracy.lib.message import render_body
from adhocracy.lib.message import send as send_message
from adhocracy.lib.base import BaseController
from adhocracy.lib.templating import render, ret_abort, ret_success
from adhocracy.lib.util import get_entity_or_abort
from adhocracy.lib import democracy
from adhocracy.model import Instance
from adhocracy.model import Membership
from adhocracy.model import Permission
from adhocracy.model import User
from adhocracy.model import UserBadge
from adhocracy.model import UserBadges
from adhocracy.model import Proposal

log = logging.getLogger(__name__)

class MassmessageBaseForm(formencode.Schema):
    allow_extra_fields = True
    subject = validators.String(max=140, not_empty=True)
    body = validators.String(min=2, not_empty=True)

class MassmessageForm(MassmessageBaseForm):
    filter_instances = forms.MessageableInstances(not_empty=True)
    filter_badges = forms.ValidUserBadges()
    sender_email = validators.String(not_empty=True)
    sender_name = validators.String(not_empty=False, if_missing=None)
    include_footer = formencode.validators.StringBoolean(if_missing=False)

class MassmessageProposalForm(MassmessageBaseForm):
    creators = validators.StringBool(not_empty=False, if_empty=False,
                                     if_missing=False)
    supporters = validators.StringBool(not_empty=False, if_empty=False,
                                       if_missing=False)
    opponents = validators.StringBool(not_empty=False, if_empty=False,
                                      if_missing=False)
    chained_validators = [
        forms.ProposalMessageNoRecipientGroup(),
    ]

def _get_options(func):
    """ Decorator that calls the functions with the following parameters:
        sender_email - Email address of the sender
        sender_name  - Name of the sender
        subject      - Subject of the message
        body         - Body of the message
        recipients   - A list of users the email is going to
    """
    @RequireInternalRequest(methods=['POST'])
    @validate(schema=MassmessageForm(), form='new')
    def wrapper(self):
        allowed_sender_options = self._get_allowed_sender_options(c.user)
        sender_email = self.form_result.get('sender_email')
        if ((sender_email not in allowed_sender_options) or
                (not allowed_sender_options[sender_email]['enabled'])):
            return ret_abort(_("Sorry, but you're not allowed to set these "
                               "message options"), code=403)
        sender_name = None
        if has('global.message'):
            sender_name = self.form_result.get('sender_name')
        if not sender_name:
            sender_name = c.user.name

        recipients = User.all_q()
        filter_instances = self.form_result.get('filter_instances')
        recipients = recipients.join(Membership).filter(
            Membership.instance_id.in_(filter_instances))
        filter_badges = self.form_result.get('filter_badges')
        if filter_badges:
            recipients = recipients.join(UserBadges,
                                         UserBadges.user_id == User.id)
            recipients = recipients.filter(
                UserBadges.badge_id.in_([fb.id for fb in filter_badges]))

        if has('global.admin'):
            include_footer = self.form_result.get('include_footer')
        else:
            include_footer = True

        if len(filter_instances) == 1:
            instance = Instance.find(filter_instances[0])
        else:
            instance = None

        return func(self,
                    self.form_result.get('subject'),
                    self.form_result.get('body'),
                    recipients.all(),
                    sender_email=allowed_sender_options[sender_email]['email'],
                    sender_name=sender_name,
                    instance=instance,
                    include_footer=include_footer,
                    )
    return wrapper

class MassmessageController(BaseController):
    """
    This deals with messages to multiple users at the same time. This will be
    will be merged with the MessageController at some point.
    """

    @classmethod
    def _get_allowed_instances(cls, user):
        """
        returns all instances in which the given user has permission to send a
        message to all users
        """
        if has('global.message'):
            return Instance.all(include_hidden=True)
        else:
            perm = Permission.find('instance.message')
            instances = [m.instance for m in user.memberships
                         if (m.instance is not None
                             and m.instance.is_authenticated
                             and perm in m.group.permissions)]
            return sorted(instances, key=lambda i: i.label)

    @classmethod
    def _get_allowed_sender_options(cls, user):
        sender_options = {
            'user': {
                'email': user.email,
                'checked': False,
                'enabled': user.is_email_activated(),
                'reason': _("Email isn't activated"),
            },
            'system': {
                'email': config.get('adhocracy.email.from'),
                'checked': False,
                'enabled': config.get_bool(
                    'allow_system_email_in_mass_messages'),
                'reason': _("Not permitted in system settings"),
            },
            'support': {
                'email': config.get('adhocracy.registration_support_email'),
                'checked': False,
                'enabled': (config.get('adhocracy.registration_support_email')
                            is not None),
                'reason': _("adhocracy.registration_support_email not set"),
            }
        }

        if sender_options['user']['enabled']:
            sender_options['user']['checked'] = True
        elif sender_options['system']['enabled']:
            sender_options['system']['checked'] = True

        return sender_options

    def new(self, id=None, errors={}, format=u'html'):

        if id is None:
            require.perm('global.message')
            template = '/massmessage/new.html'
            c.preview_url = h.base_url('/message/preview')
        else:
            c.page_instance = InstanceController._get_current_instance(id)
            require.instance.message(c.page_instance)
            template = '/instance/message.html'
            c.preview_url = h.base_url(
                '/instance/%s/message/preview' % id)

        defaults = dict(request.params)
        defaults.setdefault('include_footer', 'on')

        data = {
            'instances': self._get_allowed_instances(c.user),
            'sender_options': self._get_allowed_sender_options(c.user),
            'userbadges': UserBadge.all(instance=c.instance,
                                        include_global=True)
        }

        return htmlfill.render(render(template, data,
                                      overlay=format == u'overlay'),
                               defaults=defaults, errors=errors,
                               force_defaults=False)

    @_get_options
    def preview(self, subject, body, recipients, sender_email, sender_name,
                instance, include_footer):
        recipients_list = sorted(list(recipients), key=lambda r: r.name)
        if recipients_list:
            try:
                rendered_body = render_body(body, recipients_list[0],
                                            is_preview=True)
            except (KeyError, ValueError) as e:
                rendered_body = _('Could not render message: %s') % str(e)
        else:
            rendered_body = body

        # wrap body, but leave long words (e.g. links) intact
        rendered_body = u'\n'.join(textwrap.fill(line, break_long_words=False)
                                   for line in rendered_body.split(u'\n'))

        data = {
            'sender_email': sender_email,
            'sender_name': sender_name,
            'subject': subject,
            'body': rendered_body,
            'recipients': recipients_list,
            'recipients_count': len(recipients_list),
            'params': request.params,
            'include_footer': include_footer,
            'instance': instance,
        }
        return render('/massmessage/preview.html', data)

    @_get_options
    def create(self, subject, body, recipients, sender_email, sender_name,
               instance, include_footer):
        send_message(subject, body, c.user, recipients,
                     sender_email=sender_email,
                     sender_name=sender_name,
                     instance=instance,
                     include_footer=include_footer)
        return ret_success(
            message=_("Message sent to %d users.") % len(recipients))

    def new_proposal(self, proposal_id, errors={}, format=u'html'):
        c.proposal = get_entity_or_abort(Proposal, proposal_id)
        require.proposal.message(c.proposal)
        defaults = dict(request.params)
        return htmlfill.render(render('/massmessage/new_proposal.html',
                                      overlay=format == u'overlay'),
                               defaults=defaults, errors=errors,
                               force_defaults=False)

    @validate(schema=MassmessageProposalForm(), form='new_proposal')
    def create_proposal(self, proposal_id):
        c.proposal = get_entity_or_abort(Proposal, proposal_id)
        require.proposal.message(c.proposal)

        recipients = set()
        if self.form_result.get(u'supporters'):
            recipients.update(democracy.supporters(c.proposal.rate_poll))
        if self.form_result.get(u'opponents'):
            recipients.update(democracy.opponents(c.proposal.rate_poll))
        if self.form_result.get(u'creators'):
            recipients.update(c.proposal.get_creators())

        send_message(self.form_result.get('subject'),
                     self.form_result.get('body'),
                     c.user,
                     recipients,
                     instance=c.instance)
        h.flash(_("Message sent to %d users.") % len(recipients), 'success')
        redirect(h.entity_url(c.proposal))
<|endoftext|>


<commit_before>"""aiohttp request argument parsing module.

Example: ::

    import asyncio
    from aiohttp import web

    from webargs import fields
    from webargs.aiohttpparser import use_args

    hello_args = {
        'name': fields.Str(required=True)
    }
    @asyncio.coroutine
    @use_args(hello_args)
    def index(request, args):
        return web.Response(
            body='Hello {}'.format(args['name']).encode('utf-8')
        )

    app = web.Application()
    app.router.add_route('GET', '/', index)
"""
import typing

from aiohttp import web
from aiohttp.web import Request
from aiohttp import web_exceptions
from marshmallow import Schema, ValidationError
from marshmallow.fields import Field

from webargs import core
from webargs.core import json
from webargs.asyncparser import AsyncParser

def is_json_request(req: Request) -> bool:
    content_type = req.content_type
    return core.is_json(content_type)

class HTTPUnprocessableEntity(web.HTTPClientError):
    status_code = 422

# Mapping of status codes to exception classes
# Adapted from werkzeug
exception_map = {422: HTTPUnprocessableEntity}

def _find_exceptions() -> None:
    for name in web_exceptions.__all__:
        obj = getattr(web_exceptions, name)
        try:
            is_http_exception = issubclass(obj, web_exceptions.HTTPException)
        except TypeError:
            is_http_exception = False
        if not is_http_exception or obj.status_code is None:
            continue
        old_obj = exception_map.get(obj.status_code, None)
        if old_obj is not None and issubclass(obj, old_obj):
            continue
        exception_map[obj.status_code] = obj

# Collect all exceptions from aiohttp.web_exceptions
_find_exceptions()
del _find_exceptions

class AIOHTTPParser(AsyncParser):
    """aiohttp request argument parser."""

    __location_map__ = dict(
        match_info="parse_match_info", **core.Parser.__location_map__
    )

    def parse_querystring(self, req: Request, name: str, field: Field) -> typing.Any:
        """Pull a querystring value from the request."""
        return core.get_value(req.query, name, field)

    async def parse_form(self, req: Request, name: str, field: Field) -> typing.Any:
        """Pull a form value from the request."""
        post_data = self._cache.get("post")
        if post_data is None:
            self._cache["post"] = await req.post()
        return core.get_value(self._cache["post"], name, field)

    async def parse_json(self, req: Request, name: str, field: Field) -> typing.Any:
        """Pull a json value from the request."""
        json_data = self._cache.get("json")
        if json_data is None:
            if not (req.body_exists and is_json_request(req)):
                return core.missing
            try:
                json_data = await req.json(loads=json.loads)
            except json.JSONDecodeError as e:
                if e.doc == "":
                    return core.missing
                else:
                    return self.handle_invalid_json_error(e, req)
            self._cache["json"] = json_data
        return core.get_value(json_data, name, field, allow_many_nested=True)

    def parse_headers(self, req: Request, name: str, field: Field) -> typing.Any:
        """Pull a value from the header data."""
        return core.get_value(req.headers, name, field)

    def parse_cookies(self, req: Request, name: str, field: Field) -> typing.Any:
        """Pull a value from the cookiejar."""
        return core.get_value(req.cookies, name, field)

    def parse_files(self, req: Request, name: str, field: Field) -> None:
        raise NotImplementedError(
            "parse_files is not implemented. You may be able to use parse_form for "
            "parsing upload data."
        )

    def parse_match_info(self, req: Request, name: str, field: Field) -> typing.Any:
        """Pull a value from the request's ``match_info``."""
        return core.get_value(req.match_info, name, field)

    def get_request_from_view_args(
        self, view: typing.Callable, args: typing.Iterable, kwargs: typing.Mapping
    ) -> Request:
        """Get request object from a handler function or method. Used internally by
        ``use_args`` and ``use_kwargs``.
        """
        req = None
        for arg in args:
            if isinstance(arg, web.Request):
                req = arg
                break
            elif isinstance(arg, web.View):
                req = arg.request
                break
        assert isinstance(req, web.Request), "Request argument not found for handler"
        return req

    def handle_error(
        self,
        error: ValidationError,
        req: Request,
        schema: Schema,
        error_status_code: typing.Union[int, None] = None,
        error_headers: typing.Union[typing.Mapping[str, str], None] = None,
    ) -> typing.NoReturn:
        """Handle ValidationErrors and return a JSON response of error messages
        to the client.
        """
        error_class = exception_map.get(
            error_status_code or self.DEFAULT_VALIDATION_STATUS
        )
        if not error_class:
            raise LookupError("No exception for {0}".format(error_status_code))
        headers = error_headers
        raise error_class(
            body=json.dumps(error.messages).encode("utf-8"),
            headers=headers,
            content_type="application/json",
        )

    def handle_invalid_json_error(
        self, error: json.JSONDecodeError, req: Request, *args, **kwargs
    ) -> typing.NoReturn:
        error_class = exception_map[400]
        messages = {"json": ["Invalid JSON body."]}
        raise error_class(
            body=json.dumps(messages).encode("utf-8"), content_type="application/json"
        )

parser = AIOHTTPParser()
use_args = parser.use_args  # type: typing.Callable
use_kwargs = parser.use_kwargs  # type: typing.Callable
<commit_msg>Avoid AttributeError on Python<3.5.4<commit_after>"""aiohttp request argument parsing module.

Example: ::

    import asyncio
    from aiohttp import web

    from webargs import fields
    from webargs.aiohttpparser import use_args

    hello_args = {
        'name': fields.Str(required=True)
    }
    @asyncio.coroutine
    @use_args(hello_args)
    def index(request, args):
        return web.Response(
            body='Hello {}'.format(args['name']).encode('utf-8')
        )

    app = web.Application()
    app.router.add_route('GET', '/', index)
"""
import typing

from aiohttp import web
from aiohttp.web import Request
from aiohttp import web_exceptions
from marshmallow import Schema, ValidationError
from marshmallow.fields import Field

from webargs import core
from webargs.core import json
from webargs.asyncparser import AsyncParser

def is_json_request(req: Request) -> bool:
    content_type = req.content_type
    return core.is_json(content_type)

class HTTPUnprocessableEntity(web.HTTPClientError):
    status_code = 422

# Mapping of status codes to exception classes
# Adapted from werkzeug
exception_map = {422: HTTPUnprocessableEntity}

def _find_exceptions() -> None:
    for name in web_exceptions.__all__:
        obj = getattr(web_exceptions, name)
        try:
            is_http_exception = issubclass(obj, web_exceptions.HTTPException)
        except TypeError:
            is_http_exception = False
        if not is_http_exception or obj.status_code is None:
            continue
        old_obj = exception_map.get(obj.status_code, None)
        if old_obj is not None and issubclass(obj, old_obj):
            continue
        exception_map[obj.status_code] = obj

# Collect all exceptions from aiohttp.web_exceptions
_find_exceptions()
del _find_exceptions

class AIOHTTPParser(AsyncParser):
    """aiohttp request argument parser."""

    __location_map__ = dict(
        match_info="parse_match_info", **core.Parser.__location_map__
    )

    def parse_querystring(self, req: Request, name: str, field: Field) -> typing.Any:
        """Pull a querystring value from the request."""
        return core.get_value(req.query, name, field)

    async def parse_form(self, req: Request, name: str, field: Field) -> typing.Any:
        """Pull a form value from the request."""
        post_data = self._cache.get("post")
        if post_data is None:
            self._cache["post"] = await req.post()
        return core.get_value(self._cache["post"], name, field)

    async def parse_json(self, req: Request, name: str, field: Field) -> typing.Any:
        """Pull a json value from the request."""
        json_data = self._cache.get("json")
        if json_data is None:
            if not (req.body_exists and is_json_request(req)):
                return core.missing
            try:
                json_data = await req.json(loads=json.loads)
            except json.JSONDecodeError as e:
                if e.doc == "":
                    return core.missing
                else:
                    return self.handle_invalid_json_error(e, req)
            self._cache["json"] = json_data
        return core.get_value(json_data, name, field, allow_many_nested=True)

    def parse_headers(self, req: Request, name: str, field: Field) -> typing.Any:
        """Pull a value from the header data."""
        return core.get_value(req.headers, name, field)

    def parse_cookies(self, req: Request, name: str, field: Field) -> typing.Any:
        """Pull a value from the cookiejar."""
        return core.get_value(req.cookies, name, field)

    def parse_files(self, req: Request, name: str, field: Field) -> None:
        raise NotImplementedError(
            "parse_files is not implemented. You may be able to use parse_form for "
            "parsing upload data."
        )

    def parse_match_info(self, req: Request, name: str, field: Field) -> typing.Any:
        """Pull a value from the request's ``match_info``."""
        return core.get_value(req.match_info, name, field)

    def get_request_from_view_args(
        self, view: typing.Callable, args: typing.Iterable, kwargs: typing.Mapping
    ) -> Request:
        """Get request object from a handler function or method. Used internally by
        ``use_args`` and ``use_kwargs``.
        """
        req = None
        for arg in args:
            if isinstance(arg, web.Request):
                req = arg
                break
            elif isinstance(arg, web.View):
                req = arg.request
                break
        assert isinstance(req, web.Request), "Request argument not found for handler"
        return req

    def handle_error(
        self,
        error: ValidationError,
        req: Request,
        schema: Schema,
        error_status_code: typing.Union[int, None] = None,
        error_headers: typing.Union[typing.Mapping[str, str], None] = None,
    ) -> "typing.NoReturn":
        """Handle ValidationErrors and return a JSON response of error messages
        to the client.
        """
        error_class = exception_map.get(
            error_status_code or self.DEFAULT_VALIDATION_STATUS
        )
        if not error_class:
            raise LookupError("No exception for {0}".format(error_status_code))
        headers = error_headers
        raise error_class(
            body=json.dumps(error.messages).encode("utf-8"),
            headers=headers,
            content_type="application/json",
        )

    def handle_invalid_json_error(
        self, error: json.JSONDecodeError, req: Request, *args, **kwargs
    ) -> "typing.NoReturn":
        error_class = exception_map[400]
        messages = {"json": ["Invalid JSON body."]}
        raise error_class(
            body=json.dumps(messages).encode("utf-8"), content_type="application/json"
        )

parser = AIOHTTPParser()
use_args = parser.use_args  # type: typing.Callable
use_kwargs = parser.use_kwargs  # type: typing.Callable
<|endoftext|>


<commit_before>from django.http import HttpResponseRedirect
from django.utils.http import is_safe_url
from django.views.generic import CreateView
from .forms import BugReportForm
from django.contrib import messages

class BugReportFormView(CreateView):
    form_class = BugReportForm
    template_name = 'bug_reports/report_form_view.html'

    def get_success_url(self):
        messages.success(self.request, 'Thank you for your feedback!')

        if self.source_redirect and\
            is_safe_url(self.object.source_url) and\
            'report_problem' not in self.object.source_url:

            return self.object.source_url
        else:
            return "/"

    def post(self, request, *args, **kwargs):
        self.object = None
        form = self.get_form()
        if form.is_valid():
            report = form.save()

            report.status = 'OPEN'
            report.report_type = 'OTHER'
            report.user_agent = request.META['HTTP_USER_AGENT']

            # if the source_url came from our hidden field
            # we'll try to redirect to it if it passes 'is_safe_url()'
            self.source_redirect = True

            # try to populate these from request params
            # if they weren't submitted with the form
            if not report.source_url:
                report.source_url = request.GET.get('source_url', '')

                # if the source_url came from request.GET
                # we don't want to try and redirect
                # even if it passes 'is_safe_url()'
                self.source_redirect = False

            if not report.source:
                report.source = request.GET.get('source', '')

            report.save()
            self.object = report
            return HttpResponseRedirect(self.get_success_url())
        else:
            return self.form_invalid(form)
<commit_msg>don't fall over if user agent not set<commit_after>from django.http import HttpResponseRedirect
from django.utils.http import is_safe_url
from django.views.generic import CreateView
from .forms import BugReportForm
from django.contrib import messages

class BugReportFormView(CreateView):
    form_class = BugReportForm
    template_name = 'bug_reports/report_form_view.html'

    def get_success_url(self):
        messages.success(self.request, 'Thank you for your feedback!')

        if self.source_redirect and\
            is_safe_url(self.object.source_url) and\
            'report_problem' not in self.object.source_url:

            return self.object.source_url
        else:
            return "/"

    def post(self, request, *args, **kwargs):
        self.object = None
        form = self.get_form()
        if form.is_valid():
            report = form.save()

            report.status = 'OPEN'
            report.report_type = 'OTHER'
            try:
                report.user_agent = request.META['HTTP_USER_AGENT']
            except KeyError:
                report.user_agent = ''

            # if the source_url came from our hidden field
            # we'll try to redirect to it if it passes 'is_safe_url()'
            self.source_redirect = True

            # try to populate these from request params
            # if they weren't submitted with the form
            if not report.source_url:
                report.source_url = request.GET.get('source_url', '')

                # if the source_url came from request.GET
                # we don't want to try and redirect
                # even if it passes 'is_safe_url()'
                self.source_redirect = False

            if not report.source:
                report.source = request.GET.get('source', '')

            report.save()
            self.object = report
            return HttpResponseRedirect(self.get_success_url())
        else:
            return self.form_invalid(form)
<|endoftext|>


<commit_before>import matplotlib.pyplot as plt
import numpy as np

from simulate import popgen_simulate
from simulate_reversible import popgen_simulate_reversible

N_list = [1e2, 5*1e2, 1e3, 5*1e3, 1e4, 5*1e4, 1e5, 5*1e5, 1e6, 5*1e6, 1e7]

# parameters to replicate figure 5a of 2014 valley crossing paper
# for each param combo, they average over 500 independent runs
delta = -0.1
s = -0.085

# regime A
alpha_plus = 0.2
alpha_minus = 0.5
mu = 0.5
mutant_traits_norev = [(0.0, alpha_plus, 0),    # base pop (x)
                       (delta, mu, 0),          # 1-mutant (y)
                       (s, 0.0, 0)]             # 2-mutant (z)
mutant_traits_reversible = [(0.0, alpha_plus, 0.0),        # base pop (x)
                            (delta, mu, alpha_minus),      # 1-mutant (y)
                            (s, 0.0, 0.0)]                 # 2-mutant (z)
mutant_traits_reversible_fast = [(0.0, alpha_plus, 0.0),            # base pop (x)
                                 (delta, 10*mu, alpha_minus),      # 1-mutant (y)
                                 (s, 0.0, 0.0)]                     # 2-mutant (z)

"""
mu_0 = 1e-4
mu_0_backward = 1e-4
mu_1 = 1e-5
mutant_traits_norev = [(0.0, mu_0, 0),    # base pop (x)
                       (delta, mu_1, 0),  # 1-mutant (y)
                       (s, 0.0, 0)]       # 2-mutant (z)
mutant_traits_reversible = [(0.0, mu_0, 0.0),              # base pop (x)
                            (delta, mu_1, mu_0_backward),  # 1-mutant (y)
                            (s, 0.0, 0.0)]                 # 2-mutant (z)
mutant_traits_reversible_fast = [(0.0, mu_0, 0.0),                      # base pop (x)
                                 (delta, mu_1, 0.1*mu_0_backward),      # 1-mutant (y)
                                 (s, 0.0, 0.0)]                         # 2-mutant (z)
"""

REPEATS=10
def get_average_run(N, mutant_traits, repeats=REPEATS):
    # TODO: should also be reporting stdev or variance
    trials = [0 for i in xrange(repeats)]
    for i in xrange(repeats):
        population, t = popgen_simulate_reversible(N, mutant_traits)
        trials[i] = t
    return sum(trials)/float(repeats), np.std(trials)

# simulate
t_norev = np.zeros(len(N_list))
t_reversible = np.zeros(len(N_list))
t_fast = np.zeros(len(N_list))
sd_norev = np.zeros(len(N_list))
sd_reversible = np.zeros(len(N_list))
sd_fast = np.zeros(len(N_list))

for i, N in enumerate(N_list):
    t_norev[i], sd_norev[i] = get_average_run(N, mutant_traits_norev)
    t_reversible[i], sd_reversible[i] = get_average_run(N, mutant_traits_reversible)
    t_fast[i], sd_fast[i] = get_average_run(N, mutant_traits_reversible_fast)

# plot output
plt.errorbar(N_list, t_norev, yerr=sd_norev, fmt='--s', label='not reversible')
plt.errorbar(N_list, t_reversible, yerr=sd_reversible, fmt='--o', label='reversible')
plt.errorbar(N_list, t_fast, yerr=sd_fast, fmt='--d', label='less reversible')
ax = plt.gca()
ax.set_xscale('log')
ax.set_yscale('log')
plt.xlabel('N')
plt.ylabel('time')
plt.legend(loc='upper right')
plt.title('Mean fixation time (avg %d runs per data point)' % REPEATS)

plt.show()
<commit_msg>adjust plot label<commit_after>import matplotlib.pyplot as plt
import numpy as np

from simulate import popgen_simulate
from simulate_reversible import popgen_simulate_reversible

N_list = [1e2, 5*1e2, 1e3, 5*1e3, 1e4, 5*1e4, 1e5, 5*1e5, 1e6, 5*1e6, 1e7]

# parameters to replicate figure 5a of 2014 valley crossing paper
# for each param combo, they average over 500 independent runs
delta = -0.1
s = -0.085

# regime A
alpha_plus = 0.2
alpha_minus = 0.5
mu = 0.5
mutant_traits_norev = [(0.0, alpha_plus, 0),    # base pop (x)
                       (delta, mu, 0),          # 1-mutant (y)
                       (s, 0.0, 0)]             # 2-mutant (z)
mutant_traits_reversible = [(0.0, alpha_plus, 0.0),        # base pop (x)
                            (delta, mu, alpha_minus),      # 1-mutant (y)
                            (s, 0.0, 0.0)]                 # 2-mutant (z)
mutant_traits_reversible_fast = [(0.0, alpha_plus, 0.0),           # base pop (x)
                                 (delta, 10*mu, alpha_minus),      # 1-mutant (y)
                                 (s, 0.0, 0.0)]                    # 2-mutant (z)

"""
mu_0 = 1e-4
mu_0_backward = 1e-4
mu_1 = 1e-5
mutant_traits_norev = [(0.0, mu_0, 0),    # base pop (x)
                       (delta, mu_1, 0),  # 1-mutant (y)
                       (s, 0.0, 0)]       # 2-mutant (z)
mutant_traits_reversible = [(0.0, mu_0, 0.0),              # base pop (x)
                            (delta, mu_1, mu_0_backward),  # 1-mutant (y)
                            (s, 0.0, 0.0)]                 # 2-mutant (z)
mutant_traits_reversible_fast = [(0.0, mu_0, 0.0),                      # base pop (x)
                                 (delta, mu_1, 0.1*mu_0_backward),      # 1-mutant (y)
                                 (s, 0.0, 0.0)]                         # 2-mutant (z)
"""

REPEATS=10
def get_average_run(N, mutant_traits, repeats=REPEATS):
    # TODO: should also be reporting stdev or variance
    trials = [0 for i in xrange(repeats)]
    for i in xrange(repeats):
        population, t = popgen_simulate_reversible(N, mutant_traits)
        trials[i] = t
    return sum(trials)/float(repeats), np.std(trials)

# simulate
t_norev = np.zeros(len(N_list))
t_reversible = np.zeros(len(N_list))
t_fast = np.zeros(len(N_list))
sd_norev = np.zeros(len(N_list))
sd_reversible = np.zeros(len(N_list))
sd_fast = np.zeros(len(N_list))

for i, N in enumerate(N_list):
    t_norev[i], sd_norev[i] = get_average_run(N, mutant_traits_norev)
    t_reversible[i], sd_reversible[i] = get_average_run(N, mutant_traits_reversible)
    t_fast[i], sd_fast[i] = get_average_run(N, mutant_traits_reversible_fast)

# plot output
plt.errorbar(N_list, t_norev, yerr=sd_norev, fmt='--s', label='not reversible')
plt.errorbar(N_list, t_reversible, yerr=sd_reversible, fmt='--o', label='reversible')
plt.errorbar(N_list, t_fast, yerr=sd_fast, fmt='--d', label='faster mutation')
ax = plt.gca()
ax.set_xscale('log')
ax.set_yscale('log')
plt.xlabel('N')
plt.ylabel('time')
plt.legend(loc='upper right')
plt.title('Mean fixation time (avg %d runs per data point)' % REPEATS)

plt.show()
<|endoftext|>


<commit_before>__version__="0.10.3"

<commit_msg><commit_after>__version__="0.10.4"

<|endoftext|>


<commit_before>import os
import urllib
from thredds_crawler.crawl import Crawl

import logging
import logging.handlers
logger = logging.getLogger('thredds_crawler')
fh = logging.handlers.RotatingFileHandler('/home/testbed/iso/logs/iso_harvest.log', maxBytes=1024*1024*10, backupCount=5)
fh.setLevel(logging.DEBUG)
ch = logging.StreamHandler()
ch.setLevel(logging.DEBUG)
formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
fh.setFormatter(formatter)
ch.setFormatter(formatter)
logger.addHandler(fh)
logger.addHandler(ch)
logger.setLevel(logging.DEBUG)

SAVE_DIR="/home/testbed/iso/iso_records"

THREDDS_SERVERS = {
    "espresso1": "http://tds.marine.rutgers.edu/thredds/roms/espresso/2009_da/catalog.html",
    "espresso2": "http://tds.marine.rutgers.edu/thredds/catalog/roms/espresso/2013_da/avg_Best/catalog.html",
    "espresso3": "http://tds.marine.rutgers.edu/thredds/catalog/roms/espresso/2013_da/his_Best/catalog.html",
    "smast2": "http://www.smast.umassd.edu:8080/thredds/hindcasts.html",
    "smast3": "http://www.smast.umassd.edu:8080/thredds/archives.html",
    "comt1":   "http://comt.sura.org/thredds/comt_1_archive_summary.html",
    "comt2":   "http://comt.sura.org/thredds/comt_2_current.html",
    "useast":  "http://omgsrv1.meas.ncsu.edu:8080/thredds/catalog/fmrc/us_east/catalog.html",
    "wfs_roms_nf": "http://crow.marine.usf.edu:8080/thredds/catalog/WFS_ROMS_NF_model/catalog.html",
    "wfs_fvcom": "http://crow.marine.usf.edu:8080/thredds/fvcom_agg.html",
    "wfs_swan": "http://crow.marine.usf.edu:8080/thredds/swan_agg.html",
    "tampa_hf_radar": "http://crow.marine.usf.edu:8080/thredds/hf_radar_agg.html",
    "hycom_region1": "http://ecowatch.ncddc.noaa.gov/thredds/catalog/hycom/hycom_reg1_agg/catalog.html",
    "hycom_region6": "http://ecowatch.ncddc.noaa.gov/thredds/catalog/hycom/hycom_reg6_agg/catalog.html",
    "hycom_region7": "http://ecowatch.ncddc.noaa.gov/thredds/catalog/hycom/hycom_reg7_agg/catalog.html",
    "hycom_region17": "http://ecowatch.ncddc.noaa.gov/thredds/catalog/hycom/hycom_reg17_agg/catalog.html",
    '0': u'http://www.neracoos.org/thredds/WW3.xml',
    '1': u'http://geoport.whoi.edu/thredds/catalog/coawst_4/use/fmrc/catalog.xml',
    '10': u'http://ecowatch.ncddc.noaa.gov/thredds/catalog/ncom/ncom_reg10_agg/catalog.xml',
     '11': u'http://ecowatch.ncddc.noaa.gov/thredds/catalog/ncom/ncom_reg1_agg/catalog.xml',
     '12': u'http://ona.coas.oregonstate.edu:8080/thredds/catalog.xml',
     '13': u'http://thredds.cdip.ucsd.edu/thredds/catalog/cdip/archive/028p1/catalog.xml',
     '14': u'http://thredds.cdip.ucsd.edu/thredds/catalog/cdip/archive/128p1/catalog.xml',
     '15': u'http://thredds.cdip.ucsd.edu/thredds/catalog/cdip/archive/133p1/catalog.xml',
     '16': u'http://www.smast.umassd.edu:8080/thredds/forecasts.xml',
     '17': u'http://geoport-dev.whoi.edu/thredds/estofs_agg.xml',
     '18': u'http://colossus.dl.stevens-tech.edu/thredds/complete_latest.xml',
     '19': u'http://www.cencoos.org/thredds/catalog/gliders/bloomex/catalog.xml',
     '2': u'http://sos.maracoos.org/stable/agg_catalogs/weatherflow_agg_catalog.xml',
     '20': u'http://thredds.cdip.ucsd.edu/thredds/catalog/cdip/archive/029p1/catalog.xml',
     '21': u'http://thredds.cdip.ucsd.edu/thredds/catalog/cdip/archive/036p1/catalog.xml',
     '22': u'http://thredds.cdip.ucsd.edu/thredds/catalog/cdip/archive/071p1/catalog.xml',
    '23': u'http://thredds.cdip.ucsd.edu/thredds/catalog/cdip/archive/076p1/catalog.xml',
    '24': u'http://thredds.cdip.ucsd.edu/thredds/catalog/cdip/archive/081p1/catalog.xml',
    '25': u'http://thredds.cdip.ucsd.edu/thredds/catalog/cdip/archive/087p1/catalog.xml',
     '26': u'http://thredds.cdip.ucsd.edu/thredds/catalog/cdip/archive/088p1/catalog.xml',
     '27': u'http://thredds.cdip.ucsd.edu/thredds/catalog/cdip/archive/089p1/catalog.xml',
     '28': u'http://thredds.cdip.ucsd.edu/thredds/catalog/cdip/archive/090p1/catalog.xml',
     '29': u'http://thredds.cdip.ucsd.edu/thredds/catalog/cdip/archive/091p1/catalog.xml',
     '3': u'http://sos.maracoos.org/stable/agg_catalogs/sldmb_agg_catalog.xml',
     '30': u'http://thredds.cdip.ucsd.edu/thredds/catalog/cdip/archive/095p1/catalog.xml',
     '31': u'http://omgsrv1.meas.ncsu.edu:8080/thredds/catalog/fmrc/sabgom/catalog.xml',
     '32': u'http://ecowatch.ncddc.noaa.gov/thredds/catalog/ncom/ncom_sfc8_hind_agg/catalog.xml',
     '33': u'http://ecowatch.ncddc.noaa.gov/thredds/catalog/ncom_us_east_before_depth_change_agg/catalog.xml',
     '34': u'http://ecowatch.ncddc.noaa.gov/thredds/catalog/ncom_useast_agg_20091119_20130404/catalog.xml',
     '35': u'http://ecowatch.ncddc.noaa.gov/thredds/catalog/ncom_us_east_agg/catalog.xml',
     '36': u' http://ecowatch.ncddc.noaa.gov/thredds/catalog/ncom_amseas_agg_20091119_20130404/catalog.xml',
     '37': u' http://ecowatch.ncddc.noaa.gov/thredds/catalog/ncom_amseas_agg/catalog.xml',
     '38': u'http://hfrnet.ucsd.edu/thredds/HFRADAR_USEGC_hourly_GNOME.xml',
     '39': u'http://hfrnet.ucsd.edu/thredds/HFRADAR_USHI_hourly_GNOME.xml',
     '4': u'http://sos.maracoos.org/thredds/catalog.xml',
     '40': u'http://hfrnet.ucsd.edu/thredds/HFRADAR_PRVI_hourly_GNOME.xml',
     '41': u'http://thredds.cdip.ucsd.edu/thredds/catalog/cdip/archive/104p1/catalog.xml',
     '42': u'http://thredds.cdip.ucsd.edu/thredds/catalog/cdip/archive/105p1/catalog.xml',
     '43': u'http://opendap.co-ops.nos.noaa.gov/thredds/catalog/GBOFS/fmrc/catalog.xml',
     '44': u'http://opendap.co-ops.nos.noaa.gov/thredds/catalog/CBOFS/fmrc/catalog.xml',
     '45': u'http://opendap.co-ops.nos.noaa.gov/thredds/catalog/LOOFS/fmrc/catalog.xml',
     '46': u'http://opendap.co-ops.nos.noaa.gov/thredds/catalog/LMOFS/fmrc/catalog.xml',
     '47': u'http://opendap.co-ops.nos.noaa.gov/thredds/catalog/LEOFS/fmrc/catalog.xml',
     '48': u'http://opendap.co-ops.nos.noaa.gov/thredds/catalog/LSOFS/fmrc/catalog.xml',
     '49': u'http://opendap.co-ops.nos.noaa.gov/thredds/catalog/SJROFS/fmrc/catalog.xml',
     '5': u'http://ecowatch.ncddc.noaa.gov/thredds/catalog/ncom/ncom_reg7_agg/catalog.xml',
     '50': u'http://opendap.co-ops.nos.noaa.gov/thredds/catalog/TBOFS/fmrc/catalog.xml',
     '51': u'http://thredds.cdip.ucsd.edu/thredds/catalog/cdip/archive/139p1/catalog.xml',
     '52': u'http://thredds.cdip.ucsd.edu/thredds/catalog/cdip/archive/143p1/catalog.xml',
     '53': u'http://thredds.cdip.ucsd.edu/thredds/catalog/cdip/archive/144p1/catalog.xml',
     '54': u'http://thredds.cdip.ucsd.edu/thredds/catalog/cdip/archive/151p1/catalog.xml',
     '55': u'http://thredds.cdip.ucsd.edu/thredds/catalog/cdip/archive/152p1/catalog.xml',
     '56': u'http://thredds.cdip.ucsd.edu/thredds/catalog/cdip/archive/154p1/catalog.xml',
     '57': u'http://thredds.cdip.ucsd.edu/thredds/catalog/cdip/archive/163p1/catalog.xml',
     '58': u'http://thredds.cdip.ucsd.edu/thredds/catalog/cdip/archive/164p1/catalog.xml',
     '59': u'http://thredds.cdip.ucsd.edu/thredds/catalog/cdip/archive/165p1/catalog.xml',
     '6': u'http://ecowatch.ncddc.noaa.gov/thredds/catalog/ncom/ncom_sfc8_agg/catalog.xml',
     '60': u'http://thredds.cdip.ucsd.edu/thredds/catalog/cdip/archive/166p1/catalog.xml',
     '61': u'http://thredds.cdip.ucsd.edu/thredds/catalog/cdip/archive/167p1/catalog.xml',
     '62': u'http://thredds.cdip.ucsd.edu/thredds/catalog/cdip/archive/168p1/catalog.xml',
     '63': u'http://thredds.cdip.ucsd.edu/thredds/catalog/cdip/archive/169p1/catalog.xml',
     '64': u'http://thredds.cdip.ucsd.edu/thredds/catalog/cdip/archive/170p1/catalog.xml',
     '65': u'http://thredds.cdip.ucsd.edu/thredds/catalog/cdip/archive/172p1/catalog.xml',
     '66': u'http://thredds.cdip.ucsd.edu/thredds/catalog/cdip/archive/174p1/catalog.xml',
     '67': u'http://thredds.cdip.ucsd.edu/thredds/catalog/cdip/archive/175p1/catalog.xml',
     '68': u'http://thredds.cdip.ucsd.edu/thredds/catalog/cdip/archive/176p1/catalog.xml',
     '69': u'http://thredds.cdip.ucsd.edu/thredds/catalog/cdip/archive/179p1/catalog.xml',
     '7': u'http://ecowatch.ncddc.noaa.gov/thredds/catalog/ncom/ncom_reg6_agg/catalog.xml',
     '70': u'http://thredds.cdip.ucsd.edu/thredds/catalog/cdip/archive/180p1/catalog.xml',
     '71': u'http://thredds.cdip.ucsd.edu/thredds/catalog/cdip/archive/181p1/catalog.xml',
     '72': u'http://thredds.cdip.ucsd.edu/thredds/catalog/cdip/archive/182p1/catalog.xml',
     '73': u'http://thredds.cdip.ucsd.edu/thredds/catalog/cdip/archive/184p1/catalog.xml',
     '74': u'http://thredds.cdip.ucsd.edu/thredds/catalog/cdip/archive/185p1/catalog.xml',
     '75': u'http://thredds.cdip.ucsd.edu/thredds/catalog/cdip/archive/186p1/catalog.xml',
     '76': u'http://thredds.cdip.ucsd.edu/thredds/catalog/cdip/archive/187p1/catalog.xml',
     '77': u'http://thredds.cdip.ucsd.edu/thredds/catalog/cdip/archive/188p1/catalog.xml',
     '78': u'http://thredds.cdip.ucsd.edu/thredds/catalog/cdip/archive/190p1/catalog.xml',
     '79': u'http://thredds.cdip.ucsd.edu/thredds/catalog/cdip/archive/191p1/catalog.xml',
     '8': u'http://ecowatch.ncddc.noaa.gov/thredds/catalog/ncom/ncom_reg2_agg/catalog.xml',
     '80': u'http://thredds.cdip.ucsd.edu/thredds/catalog/cdip/archive/192p1/catalog.xml',
     '81': u'http://thredds.cdip.ucsd.edu/thredds/catalog/cdip/archive/195p1/catalog.xml',
     '82': u'http://thredds.cdip.ucsd.edu/thredds/catalog/cdip/archive/196p1/catalog.xml',
     '83': u'http://thredds.cdip.ucsd.edu/thredds/catalog/cdip/archive/197p1/catalog.xml',
     '84': u'http://thredds.cdip.ucsd.edu/thredds/catalog/cdip/archive/198p1/catalog.xml',
     '85': u'http://thredds.cdip.ucsd.edu/thredds/catalog/cdip/archive/200p1/catalog.xml',
     '86': u'http://thredds.cdip.ucsd.edu/thredds/catalog/cdip/archive/201p1/catalog.xml',
     '87': u'http://thredds.cdip.ucsd.edu/thredds/catalog/cdip/archive/203p1/catalog.xml',
     '88': u'http://thredds.cdip.ucsd.edu/thredds/catalog/cdip/archive/204p1/catalog.xml',
     '89': u'http://thredds.cdip.ucsd.edu/thredds/catalog/cdip/archive/205p1/catalog.xml',
     '9': u'http://ecowatch.ncddc.noaa.gov/thredds/catalog/ncom/ncom_reg5_agg/catalog.xml',
     '90': u'http://thredds.cdip.ucsd.edu/thredds/catalog/cdip/archive/207p1/catalog.xml',
     '91': u'http://thredds.cdip.ucsd.edu/thredds/catalog/cdip/archive/430p1/catalog.xml',
     '92': u'http://thredds.cdip.ucsd.edu/thredds/catalog/cdip/archive/431p1/catalog.xml',
     '93': u'http://thredds.cdip.ucsd.edu/thredds/catalog/cdip/archive/432p1/catalog.xml',
     '94': u'http://omgsrv1.meas.ncsu.edu:8080/thredds/catalog/fmrc/us_east/catalog.xml',
     '95': u'http://dm2.caricoos.org/thredds/catalog/swan/catalog.xml',
     '96': u'http://thredds.cdip.ucsd.edu/thredds/catalog/cdip/archive/043p1/catalog.xml'
}

for subfolder, thredds_url in THREDDS_SERVERS.items():
  logger.info("Crawling %s (%s)" % (subfolder, thredds_url))
  crawler = Crawl(thredds_url, debug=True)
  isos = [(d.id, s.get("url")) for d in crawler.datasets for s in d.services if s.get("service").lower() == "iso"]
  filefolder = os.path.join(SAVE_DIR, subfolder)
  if not os.path.exists(filefolder):
    os.makedirs(filefolder)
  for iso in isos:
    try:
      filename = iso[0].replace("/", "_") + ".iso.xml"
      filepath = os.path.join(filefolder, filename)
      logger.info("Downloading/Saving %s" % filepath)
      urllib.urlretrieve(iso[1], filepath)
    except BaseException:
      logger.exception("Error!")
<commit_msg>removing crappy catalogs<commit_after>import os
import urllib
from thredds_crawler.crawl import Crawl

import logging
import logging.handlers
logger = logging.getLogger('thredds_crawler')
fh = logging.handlers.RotatingFileHandler('/home/testbed/iso/logs/iso_harvest.log', maxBytes=1024*1024*10, backupCount=5)
fh.setLevel(logging.DEBUG)
ch = logging.StreamHandler()
ch.setLevel(logging.DEBUG)
formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
fh.setFormatter(formatter)
ch.setFormatter(formatter)
logger.addHandler(fh)
logger.addHandler(ch)
logger.setLevel(logging.DEBUG)

SAVE_DIR="/home/testbed/iso/iso_records"

THREDDS_SERVERS = {
    "espresso1": "http://tds.marine.rutgers.edu/thredds/roms/espresso/2009_da/catalog.html",
    "espresso2": "http://tds.marine.rutgers.edu/thredds/catalog/roms/espresso/2013_da/avg_Best/catalog.html",
    "espresso3": "http://tds.marine.rutgers.edu/thredds/catalog/roms/espresso/2013_da/his_Best/catalog.html",
    "smast2": "http://www.smast.umassd.edu:8080/thredds/hindcasts.html",
    "smast3": "http://www.smast.umassd.edu:8080/thredds/archives.html",
    "comt1":   "http://comt.sura.org/thredds/comt_1_archive_summary.html",
    "comt2":   "http://comt.sura.org/thredds/comt_2_current.html",
    "useast":  "http://omgsrv1.meas.ncsu.edu:8080/thredds/catalog/fmrc/us_east/catalog.html",
    "wfs_roms_nf": "http://crow.marine.usf.edu:8080/thredds/catalog/WFS_ROMS_NF_model/catalog.html",
    "wfs_fvcom": "http://crow.marine.usf.edu:8080/thredds/fvcom_agg.html",
    "wfs_swan": "http://crow.marine.usf.edu:8080/thredds/swan_agg.html",
    "tampa_hf_radar": "http://crow.marine.usf.edu:8080/thredds/hf_radar_agg.html",
    "hycom_region1": "http://ecowatch.ncddc.noaa.gov/thredds/catalog/hycom/hycom_reg1_agg/catalog.html",
    "hycom_region6": "http://ecowatch.ncddc.noaa.gov/thredds/catalog/hycom/hycom_reg6_agg/catalog.html",
    "hycom_region7": "http://ecowatch.ncddc.noaa.gov/thredds/catalog/hycom/hycom_reg7_agg/catalog.html",
    "hycom_region17": "http://ecowatch.ncddc.noaa.gov/thredds/catalog/hycom/hycom_reg17_agg/catalog.html",
    '1': u'http://geoport.whoi.edu/thredds/catalog/coawst_4/use/fmrc/catalog.xml',
    '10': u'http://ecowatch.ncddc.noaa.gov/thredds/catalog/ncom/ncom_reg10_agg/catalog.xml',
     '11': u'http://ecowatch.ncddc.noaa.gov/thredds/catalog/ncom/ncom_reg1_agg/catalog.xml',
     '12': u'http://ona.coas.oregonstate.edu:8080/thredds/catalog.xml',
     '16': u'http://www.smast.umassd.edu:8080/thredds/forecasts.xml',
     '17': u'http://geoport-dev.whoi.edu/thredds/estofs_agg.xml',
     '18': u'http://colossus.dl.stevens-tech.edu/thredds/complete_latest.xml',
     '19': u'http://www.cencoos.org/thredds/catalog/gliders/bloomex/catalog.xml',
     '31': u'http://omgsrv1.meas.ncsu.edu:8080/thredds/catalog/fmrc/sabgom/catalog.xml',
     '32': u'http://ecowatch.ncddc.noaa.gov/thredds/catalog/ncom/ncom_sfc8_hind_agg/catalog.xml',
     '33': u'http://ecowatch.ncddc.noaa.gov/thredds/catalog/ncom_us_east_before_depth_change_agg/catalog.xml',
     '34': u'http://ecowatch.ncddc.noaa.gov/thredds/catalog/ncom_useast_agg_20091119_20130404/catalog.xml',
     '35': u'http://ecowatch.ncddc.noaa.gov/thredds/catalog/ncom_us_east_agg/catalog.xml',
     '36': u' http://ecowatch.ncddc.noaa.gov/thredds/catalog/ncom_amseas_agg_20091119_20130404/catalog.xml',
     '37': u' http://ecowatch.ncddc.noaa.gov/thredds/catalog/ncom_amseas_agg/catalog.xml',
     '38': u'http://hfrnet.ucsd.edu/thredds/HFRADAR_USEGC_hourly_GNOME.xml',
     '39': u'http://hfrnet.ucsd.edu/thredds/HFRADAR_USHI_hourly_GNOME.xml',
     '4': u'http://sos.maracoos.org/thredds/catalog.xml',
     '40': u'http://hfrnet.ucsd.edu/thredds/HFRADAR_PRVI_hourly_GNOME.xml',
     '43': u'http://opendap.co-ops.nos.noaa.gov/thredds/catalog/GBOFS/fmrc/catalog.xml',
     '44': u'http://opendap.co-ops.nos.noaa.gov/thredds/catalog/CBOFS/fmrc/catalog.xml',
     '45': u'http://opendap.co-ops.nos.noaa.gov/thredds/catalog/LOOFS/fmrc/catalog.xml',
     '46': u'http://opendap.co-ops.nos.noaa.gov/thredds/catalog/LMOFS/fmrc/catalog.xml',
     '47': u'http://opendap.co-ops.nos.noaa.gov/thredds/catalog/LEOFS/fmrc/catalog.xml',
     '48': u'http://opendap.co-ops.nos.noaa.gov/thredds/catalog/LSOFS/fmrc/catalog.xml',
     '49': u'http://opendap.co-ops.nos.noaa.gov/thredds/catalog/SJROFS/fmrc/catalog.xml',
     '5': u'http://ecowatch.ncddc.noaa.gov/thredds/catalog/ncom/ncom_reg7_agg/catalog.xml',
     '50': u'http://opendap.co-ops.nos.noaa.gov/thredds/catalog/TBOFS/fmrc/catalog.xml',
     '6': u'http://ecowatch.ncddc.noaa.gov/thredds/catalog/ncom/ncom_sfc8_agg/catalog.xml',
     '7': u'http://ecowatch.ncddc.noaa.gov/thredds/catalog/ncom/ncom_reg6_agg/catalog.xml',
     '8': u'http://ecowatch.ncddc.noaa.gov/thredds/catalog/ncom/ncom_reg2_agg/catalog.xml',
     '9': u'http://ecowatch.ncddc.noaa.gov/thredds/catalog/ncom/ncom_reg5_agg/catalog.xml',
     '94': u'http://omgsrv1.meas.ncsu.edu:8080/thredds/catalog/fmrc/us_east/catalog.xml',
     '95': u'http://dm2.caricoos.org/thredds/catalog/swan/catalog.xml'
}

for subfolder, thredds_url in THREDDS_SERVERS.items():
  logger.info("Crawling %s (%s)" % (subfolder, thredds_url))
  crawler = Crawl(thredds_url, debug=True)
  isos = [(d.id, s.get("url")) for d in crawler.datasets for s in d.services if s.get("service").lower() == "iso"]
  filefolder = os.path.join(SAVE_DIR, subfolder)
  if not os.path.exists(filefolder):
    os.makedirs(filefolder)
  for iso in isos:
    try:
      filename = iso[0].replace("/", "_") + ".iso.xml"
      filepath = os.path.join(filefolder, filename)
      logger.info("Downloading/Saving %s" % filepath)
      urllib.urlretrieve(iso[1], filepath)
    except BaseException:
      logger.exception("Error!")
<|endoftext|>


<commit_before># Module containing a framework for unittesting of AMPLE modules. This framework
# serves the purpose of loading all/selected test cases and running them.
#
# Author: hlfsimko
# Date: 21-Mar-2016

import os
import unittest

AMPLE_DIR = os.sep.join(os.path.abspath(os.path.dirname(__file__)).split(os.sep)[ :-1 ])

class AMPLEUnittestFramework(object):
    """Framework to run Ample unittesting"""

    def run(self, buffer=True, cases=None, pattern="test*.py", verbosity=2):
        """main routine for running the test cases

        :buffer: display STDOUT messages
        :case: list of folders to be tested
        :pattern: test file name pattern
        :verbosity: chatty output
        """
        if cases:
            suite = self._load_for_subselection(cases, pattern)
        else:
            suite = unittest.TestLoader().discover(AMPLE_DIR,
                                                   pattern=pattern,
                                                   top_level_dir=AMPLE_DIR)
        unittest.TextTestRunner(verbosity=verbosity, buffer=buffer).run(suite)

    def _load_for_subselection(self, cases, pattern):
        suite = unittest.TestSuite()
        for case in cases:
            path = os.path.join(AMPLE_DIR, case)
            _suite = unittest.TestLoader().discover(path,
                                                    pattern=pattern,
                                                    top_level_dir=AMPLE_DIR)
            suite.addTests(_suite)
            del _suite
        return suite<commit_msg>import and whitespace formatting<commit_after># Module containing a framework for unittesting of AMPLE modules. This framework
# serves the purpose of loading all/selected test cases and running them.
#
# Author: hlfsimko
# Date: 21-Mar-2016

import os
from ample.testing.constants import AMPLE_DIR
from unittest import TestLoader, TextTestRunner, TestSuite

class AMPLEUnittestFramework(object):
    """Framework to run Ample unittesting"""

    def run(self, buffer=True, cases=None, pattern="test*.py", verbosity=2):
        """main routine for running the test cases

        :buffer: display STDOUT messages
        :case: list of folders to be tested
        :pattern: test file name pattern
        :verbosity: chatty output
        """
        if cases:
            suite = self._load_for_subselection(cases, pattern)
        else:
            suite = TestLoader().discover(AMPLE_DIR, pattern=pattern,
                                          top_level_dir=AMPLE_DIR)
        TextTestRunner(verbosity=verbosity, buffer=buffer).run(suite)

    def _load_for_subselection(self, cases, pattern):
        suite = TestSuite()
        for case in cases:
            path = os.path.join(AMPLE_DIR, case)
            _suite = TestLoader().discover(path, pattern=pattern,
                                           top_level_dir=AMPLE_DIR)
            suite.addTests(_suite)
            del _suite
        return suite<|endoftext|>


<commit_before># $Id: Loader.py 1580 2009-04-29 14:54:33Z landrgr1 $
#
#  Copyright (C) 2007-2009 Greg Landrum
#   @@ All Rights Reserved @@
#
import sqlalchemy

from rdkit import Chem
from rdkit.Chem import AllChem
from rdkit.Chem import Lipinski,Descriptors,Crippen
from rdkit.Dbase.DbConnection import DbConnect
from rdkit.Dbase import DbModule
import os

from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy import Table,Column,MetaData
from sqlalchemy import Integer,Text,String,ForeignKey,Binary,DateTime,Float
from sqlalchemy.orm import relation,mapper,sessionmaker,backref
from sqlalchemy import create_engine

decBase = declarative_base()

class Compound(decBase):
  __tablename__='molecules'
  guid=Column(Integer,primary_key=True)
  smiles=Column(Text,unique=True)
  molpkl=Column(Binary)

def RegisterSchema(dbUrl,echo=False):
  engine = create_engine(dbUrl,echo=echo)
  decBase.metadata.create_all(engine)
  maker = sessionmaker(bind=engine)
  return maker
ConnectToSchema=RegisterSchema

#set up the logger:
import rdkit.RDLogger as logging
logger = logging.logger()
logger.setLevel(logging.INFO)

def ProcessMol(session,mol,globalProps,nDone,nameProp='_Name',nameCol='compound_id',
               redraw=False,keepHs=False,
               skipProps=False,addComputedProps=False,
               skipSmiles=False):
  if not mol:
    raise ValueError,'no molecule'
  if keepHs:
    Chem.SanitizeMol(mol)
  try:
    nm = mol.GetProp(nameProp)
  except KeyError:
    nm = None
  if not nm:
    nm = 'Mol_%d'%nDone

  cmpd = Compound()
  session.add(cmpd)

  if redraw:
    AllChem.Compute2DCoords(m)

  if not skipSmiles:
    cmpd.smiles=Chem.MolToSmiles(mol,True)
  cmpd.molpkl=mol.ToBinary()
  setattr(cmpd,nameCol,nm)

  if not skipProps:
    if addComputedProps:
      cmpd.DonorCount=Lipinski.NumHDonors(mol)
      cmpd.AcceptorCount=Lipinski.NumHAcceptors(mol)
      cmpd.RotatableBondCount=Lipinski.NumRotatableBonds(mol)
      cmpd.AMW=Descriptors.MolWt(mol)
      cmpd.MolLogP=Crippen.MolLogP(mol)
    pns = list(mol.GetPropNames())
    for pi,pn in enumerate(pns):
      if pn.lower()==nameCol.lower(): continue
      pv = mol.GetProp(pn).strip()
      if globalProps.has_key(pn):
        setattr(cmpd,pn.lower(),pv)
  return cmpd

def LoadDb(suppl,dbName,nameProp='_Name',nameCol='compound_id',silent=False,
           redraw=False,errorsTo=None,keepHs=False,defaultVal='N/A',skipProps=False,
           regName='molecules',skipSmiles=False,maxRowsCached=-1,
           uniqNames=False,addComputedProps=False,lazySupplier=False,
           numForPropScan=10,startAnew=True):
  if not lazySupplier:
    nMols = len(suppl)
  else:
    nMols=-1
  if not silent:
    logger.info("Generating molecular database in file %s"%dbName)
    if not lazySupplier:
      logger.info("  Processing %d molecules"%nMols)

  globalProps = {}
  if startAnew:
    if os.path.exists(dbName):
      os.unlink(dbName)
    sIter=iter(suppl)
    setattr(Compound,nameCol.lower(),Column(nameCol.lower(),String,default=defaultVal,unique=uniqNames))
    if not skipProps:
      while numForPropScan>0:
        m = sIter.next()
        if not m: continue
        for pn in m.GetPropNames():
          if pn.lower()==nameCol.lower(): continue
          if not globalProps.has_key(pn):
            globalProps[pn]=1
            setattr(Compound,pn.lower(),Column(pn.lower(),String,default=defaultVal))
        numForPropScan-=1
      if addComputedProps:
        Compound.DonorCount=Column(Integer)
        Compound.AcceptorCount=Column(Integer)
        Compound.RotatableBondCount=Column(Integer)
        Compound.AMW=Column(Float)
        Compound.MolLogP=Column(Float)
    session=RegisterSchema('sqlite:///%s'%(dbName))()
  else:
    raise NotImplementedError,'updating existing databases is not yet supported'

  nDone = 0
  cache=[]
  for m in suppl:
    nDone +=1
    if not m:
      if errorsTo:
        if hasattr(suppl,'GetItemText'):
          d = suppl.GetItemText(nDone-1)
          errorsTo.write(d)
        else:
          logger.warning('full error file support not complete')
      continue

    cmpd=ProcessMol(session,m,globalProps,nDone,nameProp=nameProp,
                   nameCol=nameCol,redraw=redraw,
                   keepHs=keepHs,skipProps=skipProps,
                   addComputedProps=addComputedProps,skipSmiles=skipSmiles)
    if cmpd is not None:
      cache.append(cmpd)

    if not silent and not nDone%100:
      logger.info('  done %d'%nDone)
      try:
        session.commit()
      except:
        session.rollback()
        for cmpd in cache:
          if not session.query(Compound).filter(Compound.smiles==cmpd.smiles).first():
            session.add(cmpd)
        session.commit()
      cache=[]

  try:
    session.commit()
  except:
    session.rollback()
    for cmpd in cache:
      if not session.query(Compound).filter(Compound.smiles==cmpd.smiles).first():
        session.add(cmpd)
    session.commit()

if __name__=='__main__':
  import sys
  sdf =Chem.SDMolSupplier(sys.argv[1])
  db =sys.argv[2]
  LoadDb(sdf,db,addComputedProps=False)
  session = RegisterSchema('sqlite:///%s'%(db))()
  print '>>>>', len(session.query(Compound).all())
<commit_msg>make sure the smiles column isn't in the database if we aren't filling it<commit_after># $Id: Loader.py 1580 2009-04-29 14:54:33Z landrgr1 $
#
#  Copyright (C) 2007-2009 Greg Landrum
#   @@ All Rights Reserved @@
#
import sqlalchemy

from rdkit import Chem
from rdkit.Chem import AllChem
from rdkit.Chem import Lipinski,Descriptors,Crippen
from rdkit.Dbase.DbConnection import DbConnect
from rdkit.Dbase import DbModule
import os

from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy import Table,Column,MetaData
from sqlalchemy import Integer,Text,String,ForeignKey,Binary,DateTime,Float
from sqlalchemy.orm import relation,mapper,sessionmaker,backref
from sqlalchemy import create_engine

decBase = declarative_base()

class Compound(decBase):
  __tablename__='molecules'
  guid=Column(Integer,primary_key=True)
  molpkl=Column(Binary)

def RegisterSchema(dbUrl,echo=False):
  engine = create_engine(dbUrl,echo=echo)
  decBase.metadata.create_all(engine)
  maker = sessionmaker(bind=engine)
  return maker
ConnectToSchema=RegisterSchema

#set up the logger:
import rdkit.RDLogger as logging
logger = logging.logger()
logger.setLevel(logging.INFO)

def ProcessMol(session,mol,globalProps,nDone,nameProp='_Name',nameCol='compound_id',
               redraw=False,keepHs=False,
               skipProps=False,addComputedProps=False,
               skipSmiles=False):
  if not mol:
    raise ValueError,'no molecule'
  if keepHs:
    Chem.SanitizeMol(mol)
  try:
    nm = mol.GetProp(nameProp)
  except KeyError:
    nm = None
  if not nm:
    nm = 'Mol_%d'%nDone

  cmpd = Compound()
  session.add(cmpd)

  if redraw:
    AllChem.Compute2DCoords(m)

  if not skipSmiles:
    cmpd.smiles=Chem.MolToSmiles(mol,True)
  cmpd.molpkl=mol.ToBinary()
  setattr(cmpd,nameCol,nm)

  if not skipProps:
    if addComputedProps:
      cmpd.DonorCount=Lipinski.NumHDonors(mol)
      cmpd.AcceptorCount=Lipinski.NumHAcceptors(mol)
      cmpd.RotatableBondCount=Lipinski.NumRotatableBonds(mol)
      cmpd.AMW=Descriptors.MolWt(mol)
      cmpd.MolLogP=Crippen.MolLogP(mol)
    pns = list(mol.GetPropNames())
    for pi,pn in enumerate(pns):
      if pn.lower()==nameCol.lower(): continue
      pv = mol.GetProp(pn).strip()
      if globalProps.has_key(pn):
        setattr(cmpd,pn.lower(),pv)
  return cmpd

def LoadDb(suppl,dbName,nameProp='_Name',nameCol='compound_id',silent=False,
           redraw=False,errorsTo=None,keepHs=False,defaultVal='N/A',skipProps=False,
           regName='molecules',skipSmiles=False,maxRowsCached=-1,
           uniqNames=False,addComputedProps=False,lazySupplier=False,
           numForPropScan=10,startAnew=True):
  if not lazySupplier:
    nMols = len(suppl)
  else:
    nMols=-1
  if not silent:
    logger.info("Generating molecular database in file %s"%dbName)
    if not lazySupplier:
      logger.info("  Processing %d molecules"%nMols)

  globalProps = {}
  if startAnew:
    if os.path.exists(dbName):
      os.unlink(dbName)
    sIter=iter(suppl)
    setattr(Compound,nameCol.lower(),Column(nameCol.lower(),String,default=defaultVal,unique=uniqNames))
    if not skipSmiles:
      Compound.smiles = Column(Text,unique=True)
    if not skipProps:
      while numForPropScan>0:
        m = sIter.next()
        if not m: continue
        for pn in m.GetPropNames():
          if pn.lower()==nameCol.lower(): continue
          if not globalProps.has_key(pn):
            globalProps[pn]=1
            setattr(Compound,pn.lower(),Column(pn.lower(),String,default=defaultVal))
        numForPropScan-=1
      if addComputedProps:
        Compound.DonorCount=Column(Integer)
        Compound.AcceptorCount=Column(Integer)
        Compound.RotatableBondCount=Column(Integer)
        Compound.AMW=Column(Float)
        Compound.MolLogP=Column(Float)
    session=RegisterSchema('sqlite:///%s'%(dbName))()
  else:
    raise NotImplementedError,'updating existing databases is not yet supported'

  nDone = 0
  cache=[]
  for m in suppl:
    nDone +=1
    if not m:
      if errorsTo:
        if hasattr(suppl,'GetItemText'):
          d = suppl.GetItemText(nDone-1)
          errorsTo.write(d)
        else:
          logger.warning('full error file support not complete')
      continue

    cmpd=ProcessMol(session,m,globalProps,nDone,nameProp=nameProp,
                   nameCol=nameCol,redraw=redraw,
                   keepHs=keepHs,skipProps=skipProps,
                   addComputedProps=addComputedProps,skipSmiles=skipSmiles)
    if cmpd is not None:
      cache.append(cmpd)

    if not silent and not nDone%100:
      logger.info('  done %d'%nDone)
      try:
        session.commit()
      except:
        session.rollback()
        for cmpd in cache:
          if not session.query(Compound).filter(Compound.smiles==cmpd.smiles).first():
            session.add(cmpd)
        session.commit()
      cache=[]

  try:
    session.commit()
  except:
    session.rollback()
    for cmpd in cache:
      if not session.query(Compound).filter(Compound.smiles==cmpd.smiles).first():
        session.add(cmpd)
    session.commit()

if __name__=='__main__':
  import sys
  sdf =Chem.SDMolSupplier(sys.argv[1])
  db =sys.argv[2]
  LoadDb(sdf,db,addComputedProps=False)
  session = RegisterSchema('sqlite:///%s'%(db))()
  print '>>>>', len(session.query(Compound).all())
<|endoftext|>


<commit_before>from __future__ import unicode_literals

import time
import random
import statistics
from autobahn.twisted.websocket import WebSocketClientProtocol, \
    WebSocketClientFactory

stats = {}

class MyClientProtocol(WebSocketClientProtocol):

    def __init__(self, *args, **kwargs):
        WebSocketClientProtocol.__init__(self, *args, **kwargs)
        self.fingerprint = "".join(random.choice("abcdefghijklmnopqrstuvwxyz") for i in range(16))
        stats[self.fingerprint] = {}

    def onConnect(self, response):
        self.opened = time.time()
        self.sent = 0
        self.last_send = None
        self.received = 0
        self.corrupted = 0
        self.out_of_order = 0
        self.latencies = []

    def onOpen(self):
        def hello():
            if self.last_send is None:
                if self.sent >= self.factory.num_messages:
                    self.sendClose()
                    return
                self.last_send = time.time()
                self.sendMessage(("%s:%s" % (self.sent, self.fingerprint)).encode("ascii"))
                self.sent += 1
            else:
                # Wait for receipt of ping
                pass
            self.factory.reactor.callLater(1.0 / self.factory.message_rate, hello)
        hello()

    def onMessage(self, payload, isBinary):
        # Detect receive-before-send
        if self.last_send is None:
            self.corrupted += 1
            print("CRITICAL: Socket %s received before sending: %s" % (self.fingerprint, payload))
            return
        num, fingerprint = payload.decode("ascii").split(":")
        if fingerprint != self.fingerprint:
            self.corrupted += 1
        try:
            if int(num) != self.received:
                self.out_of_order += 1
        except ValueError:
            self.corrupted += 1
        self.latencies.append(time.time() - self.last_send)
        self.received += 1
        self.last_send = None

    def onClose(self, wasClean, code, reason):
        if hasattr(self, "sent"):
            stats[self.fingerprint] = {
                "sent": self.sent,
                "received": self.received,
                "corrupted": self.corrupted,
                "out_of_order": self.out_of_order,
                "latencies": self.latencies,
                "connect": True,
            }
        else:
            self.fingerprint = "".join(random.choice("abcdefghijklmnopqrstuvwxyz") for i in range(16))
            stats[self.fingerprint] = {
                "sent": 0,
                "received": 0,
                "corrupted": 0,
                "out_of_order": 0,
                "connect": False,
            }

class Benchmarker(object):
    """
    Performs benchmarks against WebSockets.
    """

    def __init__(self, url, num, concurrency, rate, messages):
        self.url = url
        self.num = num
        self.concurrency = concurrency
        self.rate = rate
        self.messages = messages
        self.factory = WebSocketClientFactory(
            args.url,
        )
        self.factory.protocol = MyClientProtocol
        self.factory.num_messages = self.messages
        self.factory.message_rate = self.rate

    def loop(self):
        self.spawn_loop()
        self.progress_loop()

    def spawn_loop(self):
        self.spawn_connections()
        reactor.callLater(0.1, self.spawn_loop)

    def progress_loop(self):
        self.print_progress()
        reactor.callLater(1, self.progress_loop)

    def spawn_connections(self):
        # Stop spawning if we did the right total number
        max_to_spawn = self.num - len(stats)
        if max_to_spawn <= 0:
            return
        # Decode connection args
        host, port = self.url.split("://")[1].split(":")
        port = int(port)
        # Only spawn enough to get up to concurrency
        open_protocols = len([x for x in stats.values() if not x])
        to_spawn = min(max(self.concurrency - open_protocols, 0), max_to_spawn)
        for _ in range(to_spawn):
            reactor.connectTCP(host, port, self.factory)

    def print_progress(self):
        open_protocols = len([x for x in stats.values() if not x])
        print("%s open, %s total" % (
            open_protocols,
            len(stats),
        ))
        if open_protocols == 0 and len(stats) >= self.num:
            reactor.stop()
            self.print_stats()

    def percentile(self, values, fraction):
        """
        Returns a percentile value (e.g. fraction = 0.95 -> 95th percentile)
        """
        values = sorted(values)
        stopat = int(len(values) * fraction)
        if stopat == len(values):
            stopat -= 1
        return values[stopat]

    def print_stats(self):
        # Collect stats together
        latencies = []
        num_good = 0
        num_incomplete = 0
        num_failed = 0
        num_corruption = 0
        num_out_of_order = 0
        for entry in stats.values():
            latencies.extend(entry.get("latencies", []))
            if not entry['connect']:
                num_failed += 1
            elif entry['sent'] != entry['received']:
                num_incomplete += 1
            elif entry['corrupted']:
                num_corruption += 1
            elif entry['out_of_order']:
                num_out_of_order += 1
            else:
                num_good += 1
        # Some analysis on latencies
        latency_mean = statistics.mean(latencies)
        latency_median = statistics.median(latencies)
        latency_stdev = statistics.stdev(latencies)
        latency_95 = self.percentile(latencies, 0.95)
        latency_99 = self.percentile(latencies, 0.99)
        # Print results
        print("-------")
        print("Sockets opened: %s" % len(stats))
        print("Latency stats: Mean %.3fs  Median %.3fs  Stdev %.3f  95%% %.3fs  95%% %.3fs" % (
            latency_mean,
            latency_median,
            latency_stdev,
            latency_95,
            latency_99,
        ))
        print("Good sockets: %s (%.2f%%)" % (num_good, (float(num_good) / len(stats))*100))
        print("Incomplete sockets: %s (%.2f%%)" % (num_incomplete, (float(num_incomplete) / len(stats))*100))
        print("Corrupt sockets: %s (%.2f%%)" % (num_corruption, (float(num_corruption) / len(stats))*100))
        print("Out of order sockets: %s (%.2f%%)" % (num_out_of_order, (float(num_out_of_order) / len(stats))*100))
        print("Failed to connect: %s (%.2f%%)" % (num_failed, (float(num_failed) / len(stats))*100))

if __name__ == '__main__':

    import sys
    import argparse

    from twisted.python import log
    from twisted.internet import reactor

#    log.startLogging(sys.stdout)

    parser = argparse.ArgumentParser()
    parser.add_argument("url")
    parser.add_argument("-n", "--num", type=int, default=100, help="Total number of sockets to open")
    parser.add_argument("-c", "--concurrency", type=int, default=10, help="Number of sockets to open at once")
    parser.add_argument("-r", "--rate", type=float, default=1, help="Number of messages to send per socket per second")
    parser.add_argument("-m", "--messages", type=int, default=5, help="Number of messages to send per socket before close")
    args = parser.parse_args()

    benchmarker = Benchmarker(
        url=args.url,
        num=args.num,
        concurrency=args.concurrency,
        rate=args.rate,
        messages=args.messages,
    )
    benchmarker.loop()
    reactor.run()
<commit_msg>Don't try and spawn all sockets instantly<commit_after>from __future__ import unicode_literals

import time
import random
import statistics
from autobahn.twisted.websocket import WebSocketClientProtocol, \
    WebSocketClientFactory

stats = {}

class MyClientProtocol(WebSocketClientProtocol):

    def __init__(self, *args, **kwargs):
        WebSocketClientProtocol.__init__(self, *args, **kwargs)
        self.fingerprint = "".join(random.choice("abcdefghijklmnopqrstuvwxyz") for i in range(16))
        stats[self.fingerprint] = {}

    def onConnect(self, response):
        self.opened = time.time()
        self.sent = 0
        self.last_send = None
        self.received = 0
        self.corrupted = 0
        self.out_of_order = 0
        self.latencies = []

    def onOpen(self):
        def hello():
            if self.last_send is None:
                if self.sent >= self.factory.num_messages:
                    self.sendClose()
                    return
                self.last_send = time.time()
                self.sendMessage(("%s:%s" % (self.sent, self.fingerprint)).encode("ascii"))
                self.sent += 1
            else:
                # Wait for receipt of ping
                pass
            self.factory.reactor.callLater(1.0 / self.factory.message_rate, hello)
        hello()

    def onMessage(self, payload, isBinary):
        # Detect receive-before-send
        if self.last_send is None:
            self.corrupted += 1
            print("CRITICAL: Socket %s received before sending: %s" % (self.fingerprint, payload))
            return
        num, fingerprint = payload.decode("ascii").split(":")
        if fingerprint != self.fingerprint:
            self.corrupted += 1
        try:
            if int(num) != self.received:
                self.out_of_order += 1
        except ValueError:
            self.corrupted += 1
        self.latencies.append(time.time() - self.last_send)
        self.received += 1
        self.last_send = None

    def onClose(self, wasClean, code, reason):
        if hasattr(self, "sent"):
            stats[self.fingerprint] = {
                "sent": self.sent,
                "received": self.received,
                "corrupted": self.corrupted,
                "out_of_order": self.out_of_order,
                "latencies": self.latencies,
                "connect": True,
            }
        else:
            self.fingerprint = "".join(random.choice("abcdefghijklmnopqrstuvwxyz") for i in range(16))
            stats[self.fingerprint] = {
                "sent": 0,
                "received": 0,
                "corrupted": 0,
                "out_of_order": 0,
                "connect": False,
            }

class Benchmarker(object):
    """
    Performs benchmarks against WebSockets.
    """

    def __init__(self, url, num, concurrency, rate, messages):
        self.url = url
        self.num = num
        self.concurrency = concurrency
        self.rate = rate
        self.messages = messages
        self.factory = WebSocketClientFactory(
            args.url,
        )
        self.factory.protocol = MyClientProtocol
        self.factory.num_messages = self.messages
        self.factory.message_rate = self.rate

    def loop(self):
        self.spawn_loop()
        self.progress_loop()

    def spawn_loop(self):
        self.spawn_connections()
        reactor.callLater(0.1, self.spawn_loop)

    def progress_loop(self):
        self.print_progress()
        reactor.callLater(1, self.progress_loop)

    def spawn_connections(self):
        # Stop spawning if we did the right total number
        max_to_spawn = self.num - len(stats)
        if max_to_spawn <= 0:
            return
        # Don't spawn too many at once
        max_to_spawn = min(max_to_spawn, 10)
        # Decode connection args
        host, port = self.url.split("://")[1].split(":")
        port = int(port)
        # Only spawn enough to get up to concurrency
        open_protocols = len([x for x in stats.values() if not x])
        to_spawn = min(max(self.concurrency - open_protocols, 0), max_to_spawn)
        for _ in range(to_spawn):
            reactor.connectTCP(host, port, self.factory)

    def print_progress(self):
        open_protocols = len([x for x in stats.values() if not x])
        print("%s open, %s total" % (
            open_protocols,
            len(stats),
        ))
        if open_protocols == 0 and len(stats) >= self.num:
            reactor.stop()
            self.print_stats()

    def percentile(self, values, fraction):
        """
        Returns a percentile value (e.g. fraction = 0.95 -> 95th percentile)
        """
        values = sorted(values)
        stopat = int(len(values) * fraction)
        if stopat == len(values):
            stopat -= 1
        return values[stopat]

    def print_stats(self):
        # Collect stats together
        latencies = []
        num_good = 0
        num_incomplete = 0
        num_failed = 0
        num_corruption = 0
        num_out_of_order = 0
        for entry in stats.values():
            latencies.extend(entry.get("latencies", []))
            if not entry['connect']:
                num_failed += 1
            elif entry['sent'] != entry['received']:
                num_incomplete += 1
            elif entry['corrupted']:
                num_corruption += 1
            elif entry['out_of_order']:
                num_out_of_order += 1
            else:
                num_good += 1
        # Some analysis on latencies
        latency_mean = statistics.mean(latencies)
        latency_median = statistics.median(latencies)
        latency_stdev = statistics.stdev(latencies)
        latency_95 = self.percentile(latencies, 0.95)
        latency_99 = self.percentile(latencies, 0.99)
        # Print results
        print("-------")
        print("Sockets opened: %s" % len(stats))
        print("Latency stats: Mean %.3fs  Median %.3fs  Stdev %.3f  95%% %.3fs  95%% %.3fs" % (
            latency_mean,
            latency_median,
            latency_stdev,
            latency_95,
            latency_99,
        ))
        print("Good sockets: %s (%.2f%%)" % (num_good, (float(num_good) / len(stats))*100))
        print("Incomplete sockets: %s (%.2f%%)" % (num_incomplete, (float(num_incomplete) / len(stats))*100))
        print("Corrupt sockets: %s (%.2f%%)" % (num_corruption, (float(num_corruption) / len(stats))*100))
        print("Out of order sockets: %s (%.2f%%)" % (num_out_of_order, (float(num_out_of_order) / len(stats))*100))
        print("Failed to connect: %s (%.2f%%)" % (num_failed, (float(num_failed) / len(stats))*100))

if __name__ == '__main__':

    import sys
    import argparse

    from twisted.python import log
    from twisted.internet import reactor

#    log.startLogging(sys.stdout)

    parser = argparse.ArgumentParser()
    parser.add_argument("url")
    parser.add_argument("-n", "--num", type=int, default=100, help="Total number of sockets to open")
    parser.add_argument("-c", "--concurrency", type=int, default=10, help="Number of sockets to open at once")
    parser.add_argument("-r", "--rate", type=float, default=1, help="Number of messages to send per socket per second")
    parser.add_argument("-m", "--messages", type=int, default=5, help="Number of messages to send per socket before close")
    args = parser.parse_args()

    benchmarker = Benchmarker(
        url=args.url,
        num=args.num,
        concurrency=args.concurrency,
        rate=args.rate,
        messages=args.messages,
    )
    benchmarker.loop()
    reactor.run()
<|endoftext|>