<commit_before># coding: utf-8
import logging
from collections import defaultdict

from django import forms
from django.utils.translation import ugettext_lazy as _

from sentry.plugins.bases import notify
from sentry.http import safe_urlopen
from sentry.utils.safe import safe_execute

from . import __version__, __doc__ as package_doc

class TelegramNotificationsOptionsForm(notify.NotificationConfigurationForm):
    api_token = forms.CharField(
        label=_('BotAPI token'),
        widget=forms.TextInput(attrs={'placeholder': '123456:ABC-DEF1234ghIkl-zyx57W2v1u123ew11'}),
        help_text=_('Read more: https://core.telegram.org/bots/api#authorizing-your-bot'),
    )
    receivers = forms.CharField(
        label=_('Receivers'),
        widget=forms.Textarea(attrs={'class': 'span6'}),
        help_text=_('Enter receivers IDs (one per line). Personal messages, group chats and channels also available.'))

    message_template = forms.CharField(
        label=_('Message template'),
        widget=forms.Textarea(attrs={'class': 'span4'}),
        help_text=_('Set in standard python\'s {}-format convention, available names are: '
                    '{project_name}, {url}, {title}, {message}, {tag[%your_tag%]}'),
        initial='*[Sentry]* {project_name} {tag[level]}: *{title}*\n```{message}```\n{url}'
    )

class TelegramNotificationsPlugin(notify.NotificationPlugin):
    title = 'Telegram Notifications'
    slug = 'sentry_telegram'
    description = package_doc
    version = __version__
    author = 'Viacheslav Butorov'
    author_url = 'https://github.com/butorov/sentry-telegram'
    resource_links = [
        ('Bug Tracker', 'https://github.com/butorov/sentry-telegram/issues'),
        ('Source', 'https://github.com/butorov/sentry-telegram'),
    ]

    conf_key = 'sentry_telegram'
    conf_title = title

    project_conf_form = TelegramNotificationsOptionsForm

    logger = logging.getLogger('sentry.plugins.sentry_telegram')

    def is_configured(self, project, **kwargs):
        return bool(self.get_option('api_token', project) and self.get_option('receivers', project))

    def get_config(self, project, **kwargs):
        return [
            {
                'name': 'api_token',
                'label': 'BotAPI token',
                'type': 'text',
                'help': 'Read more: https://core.telegram.org/bots/api#authorizing-your-bot',
                'placeholder': '123456:ABC-DEF1234ghIkl-zyx57W2v1u123ew11',
                'validators': [],
                'required': True,
            },
            {
                'name': 'receivers',
                'label': 'Receivers',
                'type': 'textarea',
                'help': 'Enter receivers IDs (one per line). Personal messages, group chats and channels also available.',
                'validators': [],
                'required': True,
            },
            {
                'name': 'message_template',
                'label': 'Message Template',
                'type': 'textarea',
                'help': 'Set in standard python\'s {}-format convention, available names are: '
                    '{project_name}, {url}, {title}, {message}, {tag[%your_tag%]}. Undefined tags will be shown as [NA]',
                'validators': [],
                'required': True,
                'default': '*[Sentry]* {project_name} {tag[level]}: *{title}*\n```{message}```\n{url}'
            },
        ]

    def build_message(self, group, event):
        the_tags = defaultdict(lambda: '[NA]')
        the_tags.update({k:v for k, v in event.tags})
        names = {
            'title': event.title,
            'tag': the_tags,
            'message': event.message,
            'project_name': group.project.name,
            'url': group.get_absolute_url(),
        }

        template = self.get_message_template(group.project)

        text = template.format(**names)

        return {
            'text': text,
            'parse_mode': 'Markdown',
        }

    def build_url(self, project):
        return 'https://api.telegram.org/bot%s/sendMessage' % self.get_option('api_token', project)

    def get_message_template(self, project):
        return self.get_option('message_template', project)

    def get_receivers(self, project):
        receivers = self.get_option('receivers', project)
        if not receivers:
            return []
        return filter(bool, receivers.strip().splitlines())

    def send_message(self, url, payload, receiver):
        payload['chat_id'] = receiver
        self.logger.debug('Sending message to %s ' % receiver)
        response = safe_urlopen(
            method='POST',
            url=url,
            json=payload,
        )
        self.logger.debug('Response code: %s, content: %s' % (response.status_code, response.content))

    def notify_users(self, group, event, fail_silently=False):
        self.logger.debug('Received notification for event: %s' % event)
        receivers = self.get_receivers(group.project)
        self.logger.debug('for receivers: %s' % ', '.join(receivers or ()))
        payload = self.build_message(group, event)
        self.logger.debug('Built payload: %s' % payload)
        url = self.build_url(group.project)
        self.logger.debug('Built url: %s' % url)
        for receiver in receivers:
            safe_execute(self.send_message, url, payload, receiver, _with_transaction=False)
<commit_msg>Support new sentry release (notify_users requires triggering_rules param)<commit_after># coding: utf-8
import logging
from collections import defaultdict

from django import forms
from django.utils.translation import ugettext_lazy as _

from sentry.plugins.bases import notify
from sentry.http import safe_urlopen
from sentry.utils.safe import safe_execute

from . import __version__, __doc__ as package_doc

class TelegramNotificationsOptionsForm(notify.NotificationConfigurationForm):
    api_token = forms.CharField(
        label=_('BotAPI token'),
        widget=forms.TextInput(attrs={'placeholder': '123456:ABC-DEF1234ghIkl-zyx57W2v1u123ew11'}),
        help_text=_('Read more: https://core.telegram.org/bots/api#authorizing-your-bot'),
    )
    receivers = forms.CharField(
        label=_('Receivers'),
        widget=forms.Textarea(attrs={'class': 'span6'}),
        help_text=_('Enter receivers IDs (one per line). Personal messages, group chats and channels also available.'))

    message_template = forms.CharField(
        label=_('Message template'),
        widget=forms.Textarea(attrs={'class': 'span4'}),
        help_text=_('Set in standard python\'s {}-format convention, available names are: '
                    '{project_name}, {url}, {title}, {message}, {tag[%your_tag%]}'),
        initial='*[Sentry]* {project_name} {tag[level]}: *{title}*\n```{message}```\n{url}'
    )

class TelegramNotificationsPlugin(notify.NotificationPlugin):
    title = 'Telegram Notifications'
    slug = 'sentry_telegram'
    description = package_doc
    version = __version__
    author = 'Viacheslav Butorov'
    author_url = 'https://github.com/butorov/sentry-telegram'
    resource_links = [
        ('Bug Tracker', 'https://github.com/butorov/sentry-telegram/issues'),
        ('Source', 'https://github.com/butorov/sentry-telegram'),
    ]

    conf_key = 'sentry_telegram'
    conf_title = title

    project_conf_form = TelegramNotificationsOptionsForm

    logger = logging.getLogger('sentry.plugins.sentry_telegram')

    def is_configured(self, project, **kwargs):
        return bool(self.get_option('api_token', project) and self.get_option('receivers', project))

    def get_config(self, project, **kwargs):
        return [
            {
                'name': 'api_token',
                'label': 'BotAPI token',
                'type': 'text',
                'help': 'Read more: https://core.telegram.org/bots/api#authorizing-your-bot',
                'placeholder': '123456:ABC-DEF1234ghIkl-zyx57W2v1u123ew11',
                'validators': [],
                'required': True,
            },
            {
                'name': 'receivers',
                'label': 'Receivers',
                'type': 'textarea',
                'help': 'Enter receivers IDs (one per line). Personal messages, group chats and channels also available.',
                'validators': [],
                'required': True,
            },
            {
                'name': 'message_template',
                'label': 'Message Template',
                'type': 'textarea',
                'help': 'Set in standard python\'s {}-format convention, available names are: '
                    '{project_name}, {url}, {title}, {message}, {tag[%your_tag%]}. Undefined tags will be shown as [NA]',
                'validators': [],
                'required': True,
                'default': '*[Sentry]* {project_name} {tag[level]}: *{title}*\n```{message}```\n{url}'
            },
        ]

    def build_message(self, group, event):
        the_tags = defaultdict(lambda: '[NA]')
        the_tags.update({k:v for k, v in event.tags})
        names = {
            'title': event.title,
            'tag': the_tags,
            'message': event.message,
            'project_name': group.project.name,
            'url': group.get_absolute_url(),
        }

        template = self.get_message_template(group.project)

        text = template.format(**names)

        return {
            'text': text,
            'parse_mode': 'Markdown',
        }

    def build_url(self, project):
        return 'https://api.telegram.org/bot%s/sendMessage' % self.get_option('api_token', project)

    def get_message_template(self, project):
        return self.get_option('message_template', project)

    def get_receivers(self, project):
        receivers = self.get_option('receivers', project)
        if not receivers:
            return []
        return filter(bool, receivers.strip().splitlines())

    def send_message(self, url, payload, receiver):
        payload['chat_id'] = receiver
        self.logger.debug('Sending message to %s ' % receiver)
        response = safe_urlopen(
            method='POST',
            url=url,
            json=payload,
        )
        self.logger.debug('Response code: %s, content: %s' % (response.status_code, response.content))

    def notify_users(self, group, event, triggering_rules, fail_silently=False):
        self.logger.debug('Received notification for event: %s' % event)
        receivers = self.get_receivers(group.project)
        self.logger.debug('for receivers: %s' % ', '.join(receivers or ()))
        payload = self.build_message(group, event)
        self.logger.debug('Built payload: %s' % payload)
        url = self.build_url(group.project)
        self.logger.debug('Built url: %s' % url)
        for receiver in receivers:
            safe_execute(self.send_message, url, payload, receiver, _with_transaction=False)
<|endoftext|>


<commit_before># Microsoft Azure Linux Agent
#
# Copyright 2014 Microsoft Corporation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# Requires Python 2.4+ and Openssl 1.0+
#

import time

import azurelinuxagent.common.conf as conf
import azurelinuxagent.common.logger as logger
from azurelinuxagent.common.exception import HttpError
from azurelinuxagent.common.future import httpclient, urlparse

"""
REST api util functions
"""

RETRY_WAITING_INTERVAL = 10
secure_warning = True

def _parse_url(url):
    o = urlparse(url)
    rel_uri = o.path
    if o.fragment:
        rel_uri = "{0}#{1}".format(rel_uri, o.fragment)
    if o.query:
        rel_uri = "{0}?{1}".format(rel_uri, o.query)
    secure = False
    if o.scheme.lower() == "https":
        secure = True
    return o.hostname, o.port, secure, rel_uri

def get_http_proxy():
    """
    Get http_proxy and https_proxy from environment variables.
    Username and password is not supported now.
    """
    host = conf.get_httpproxy_host()
    port = conf.get_httpproxy_port()
    return host, port

def _http_request(method, host, rel_uri, port=None, data=None, secure=False,
                  headers=None, proxy_host=None, proxy_port=None):
    url, conn = None, None
    if secure:
        port = 443 if port is None else port
        if proxy_host is not None and proxy_port is not None:
            conn = httpclient.HTTPSConnection(proxy_host,
                                              proxy_port,
                                              timeout=10)
            conn.set_tunnel(host, port)
            # If proxy is used, full url is needed.
            url = "https://{0}:{1}{2}".format(host, port, rel_uri)
        else:
            conn = httpclient.HTTPSConnection(host,
                                              port,
                                              timeout=10)
            url = rel_uri
    else:
        port = 80 if port is None else port
        if proxy_host is not None and proxy_port is not None:
            conn = httpclient.HTTPConnection(proxy_host,
                                             proxy_port,
                                             timeout=10)
            # If proxy is used, full url is needed.
            url = "http://{0}:{1}{2}".format(host, port, rel_uri)
        else:
            conn = httpclient.HTTPConnection(host,
                                             port,
                                             timeout=10)
            url = rel_uri

    logger.verbose("HTTP connection [{0}] [{1}] [{2}] [{3}]",
                   method,
                   url,
                   data,
                   headers)

    headers = {} if headers is None else headers
    conn.request(method=method, url=url, body=data, headers=headers)
    resp = conn.getresponse()
    return resp

def http_request(method, url, data, headers=None, max_retry=3,
                 chk_proxy=False):
    """
    Sending http request to server
    On error, sleep 10 and retry max_retry times.
    """
    host, port, secure, rel_uri = _parse_url(url)
    global secure_warning

    # Check proxy
    proxy_host, proxy_port = (None, None)
    if chk_proxy:
        proxy_host, proxy_port = get_http_proxy()

    # If httplib module is not built with ssl support. Fallback to http
    if secure and not hasattr(httpclient, "HTTPSConnection"):
        secure = False
        if secure_warning:
            logger.warn("httplib is not built with ssl support")
            secure_warning = False

    # If httplib module doesn't support https tunnelling. Fallback to http
    if secure and proxy_host is not None and proxy_port is not None \
            and not hasattr(httpclient.HTTPSConnection, "set_tunnel"):
        secure = False
        if secure_warning:
            logger.warn("httplib does not support https tunnelling "
                        "(new in python 2.7)")
            secure_warning = False

    if proxy_host or proxy_port:
        logger.verbose("HTTP proxy: [{0}:{1}]", proxy_host, proxy_port)

    retry_msg = ''
    log_msg = "HTTP {0}".format(method)
    for retry in range(0, max_retry):
        retry_interval = RETRY_WAITING_INTERVAL
        try:
            resp = _http_request(method,
                                 host,
                                 rel_uri,
                                 port=port,
                                 data=data,
                                 secure=secure,
                                 headers=headers,
                                 proxy_host=proxy_host,
                                 proxy_port=proxy_port)
            logger.verbose("HTTP response status: [{0}]", resp.status)
            return resp
        except httpclient.HTTPException as e:
            retry_msg = 'HTTP exception: {0} {1}'.format(log_msg, e)
            retry_interval = 5
        except IOError as e:
            retry_msg = 'IO error: {0} {1}'.format(log_msg, e)
            retry_interval = 0
            max_retry = 0

        if retry < max_retry:
            logger.info("Retry [{0}/{1} - {3}]",
                        retry+1,
                        max_retry,
                        retry_interval,
                        retry_msg)
            time.sleep(retry_interval)

    raise HttpError("{0} failed".format(log_msg))

def http_get(url, headers=None, max_retry=3, chk_proxy=False):
    return http_request("GET", url, data=None, headers=headers,
                        max_retry=max_retry, chk_proxy=chk_proxy)

def http_head(url, headers=None, max_retry=3, chk_proxy=False):
    return http_request("HEAD", url, None, headers=headers,
                        max_retry=max_retry, chk_proxy=chk_proxy)

def http_post(url, data, headers=None, max_retry=3, chk_proxy=False):
    return http_request("POST", url, data, headers=headers,
                        max_retry=max_retry, chk_proxy=chk_proxy)

def http_put(url, data, headers=None, max_retry=3, chk_proxy=False):
    return http_request("PUT", url, data, headers=headers,
                        max_retry=max_retry, chk_proxy=chk_proxy)

def http_delete(url, headers=None, max_retry=3, chk_proxy=False):
    return http_request("DELETE", url, None, headers=headers,
                        max_retry=max_retry, chk_proxy=chk_proxy)

# End REST api util functions
<commit_msg>retry once when network unreachable (#609)<commit_after># Microsoft Azure Linux Agent
#
# Copyright 2014 Microsoft Corporation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# Requires Python 2.4+ and Openssl 1.0+
#

import time

import azurelinuxagent.common.conf as conf
import azurelinuxagent.common.logger as logger
from azurelinuxagent.common.exception import HttpError
from azurelinuxagent.common.future import httpclient, urlparse

"""
REST api util functions
"""

RETRY_WAITING_INTERVAL = 10
secure_warning = True

def _parse_url(url):
    o = urlparse(url)
    rel_uri = o.path
    if o.fragment:
        rel_uri = "{0}#{1}".format(rel_uri, o.fragment)
    if o.query:
        rel_uri = "{0}?{1}".format(rel_uri, o.query)
    secure = False
    if o.scheme.lower() == "https":
        secure = True
    return o.hostname, o.port, secure, rel_uri

def get_http_proxy():
    """
    Get http_proxy and https_proxy from environment variables.
    Username and password is not supported now.
    """
    host = conf.get_httpproxy_host()
    port = conf.get_httpproxy_port()
    return host, port

def _http_request(method, host, rel_uri, port=None, data=None, secure=False,
                  headers=None, proxy_host=None, proxy_port=None):
    url, conn = None, None
    if secure:
        port = 443 if port is None else port
        if proxy_host is not None and proxy_port is not None:
            conn = httpclient.HTTPSConnection(proxy_host,
                                              proxy_port,
                                              timeout=10)
            conn.set_tunnel(host, port)
            # If proxy is used, full url is needed.
            url = "https://{0}:{1}{2}".format(host, port, rel_uri)
        else:
            conn = httpclient.HTTPSConnection(host,
                                              port,
                                              timeout=10)
            url = rel_uri
    else:
        port = 80 if port is None else port
        if proxy_host is not None and proxy_port is not None:
            conn = httpclient.HTTPConnection(proxy_host,
                                             proxy_port,
                                             timeout=10)
            # If proxy is used, full url is needed.
            url = "http://{0}:{1}{2}".format(host, port, rel_uri)
        else:
            conn = httpclient.HTTPConnection(host,
                                             port,
                                             timeout=10)
            url = rel_uri

    logger.verbose("HTTP connection [{0}] [{1}] [{2}] [{3}]",
                   method,
                   url,
                   data,
                   headers)

    headers = {} if headers is None else headers
    conn.request(method=method, url=url, body=data, headers=headers)
    resp = conn.getresponse()
    return resp

def http_request(method, url, data, headers=None, max_retry=3,
                 chk_proxy=False):
    """
    Sending http request to server
    On error, sleep 10 and retry max_retry times.
    """
    host, port, secure, rel_uri = _parse_url(url)
    global secure_warning

    # Check proxy
    proxy_host, proxy_port = (None, None)
    if chk_proxy:
        proxy_host, proxy_port = get_http_proxy()

    # If httplib module is not built with ssl support. Fallback to http
    if secure and not hasattr(httpclient, "HTTPSConnection"):
        secure = False
        if secure_warning:
            logger.warn("httplib is not built with ssl support")
            secure_warning = False

    # If httplib module doesn't support https tunnelling. Fallback to http
    if secure and proxy_host is not None and proxy_port is not None \
            and not hasattr(httpclient.HTTPSConnection, "set_tunnel"):
        secure = False
        if secure_warning:
            logger.warn("httplib does not support https tunnelling "
                        "(new in python 2.7)")
            secure_warning = False

    if proxy_host or proxy_port:
        logger.verbose("HTTP proxy: [{0}:{1}]", proxy_host, proxy_port)

    retry_msg = ''
    log_msg = "HTTP {0}".format(method)
    for retry in range(0, max_retry):
        retry_interval = RETRY_WAITING_INTERVAL
        try:
            resp = _http_request(method,
                                 host,
                                 rel_uri,
                                 port=port,
                                 data=data,
                                 secure=secure,
                                 headers=headers,
                                 proxy_host=proxy_host,
                                 proxy_port=proxy_port)
            logger.verbose("HTTP response status: [{0}]", resp.status)
            return resp
        except httpclient.HTTPException as e:
            retry_msg = 'HTTP exception: {0} {1}'.format(log_msg, e)
            retry_interval = 5
        except IOError as e:
            retry_msg = 'IO error: {0} {1}'.format(log_msg, e)
            # error 101: network unreachable; when the adapter resets we may
            # see this transient error for a short time, retry once.
            if e.errno == 101:
                retry_interval = RETRY_WAITING_INTERVAL
                max_retry = 1
            else:
                retry_interval = 0
                max_retry = 0

        if retry < max_retry:
            logger.info("Retry [{0}/{1} - {3}]",
                        retry+1,
                        max_retry,
                        retry_interval,
                        retry_msg)
            time.sleep(retry_interval)

    raise HttpError("{0} failed".format(log_msg))

def http_get(url, headers=None, max_retry=3, chk_proxy=False):
    return http_request("GET", url, data=None, headers=headers,
                        max_retry=max_retry, chk_proxy=chk_proxy)

def http_head(url, headers=None, max_retry=3, chk_proxy=False):
    return http_request("HEAD", url, None, headers=headers,
                        max_retry=max_retry, chk_proxy=chk_proxy)

def http_post(url, data, headers=None, max_retry=3, chk_proxy=False):
    return http_request("POST", url, data, headers=headers,
                        max_retry=max_retry, chk_proxy=chk_proxy)

def http_put(url, data, headers=None, max_retry=3, chk_proxy=False):
    return http_request("PUT", url, data, headers=headers,
                        max_retry=max_retry, chk_proxy=chk_proxy)

def http_delete(url, headers=None, max_retry=3, chk_proxy=False):
    return http_request("DELETE", url, None, headers=headers,
                        max_retry=max_retry, chk_proxy=chk_proxy)

# End REST api util functions
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""
Copyright (c) 2018 by Tobias Houska
This file is part of Statistical Parameter Optimization Tool for Python (SPOTPY).
:author: Philipp Kraft

A collection of helper functions to describe spotpy setups

Usage:

>>> spotpy.describe.sampler(sampler)
>>> spotpy.describe.setup(model)
"""
from __future__ import division, absolute_import, unicode_literals
import sys
from .parameter import get_parameters_from_setup
from .algorithms._algorithm import _algorithm
if sys.version_info[0] >= 3:
    from inspect import getdoc as _getdoc
    unicode = str
else:
    def _getdoc(obj):
        u = obj.__doc__
        try:
            return u'\n'.join(l.strip() for l in u.split(u'\n') if l.strip())
        except UnicodeDecodeError:
            raise AssertionError(
                '{}: Docstring uses unicode but {} misses the line ``from __future__ import unicode_literals``'
                .format(obj, type(obj).__module__)
                )

try:
    from docutils.core import publish_string
except ImportError:
    publish_string = None

def describe(obj):
    """
    Returns a long string description of a sampler with its model
    :param obj: A sampler
    :return: str
    """
    return 'Sampler:\n--------\n{}\n\nModel:\n------\n{}'.format(sampler(obj), setup(obj.setup))

def sampler(obj):
    """
    Returns a string representation of the sampler.
    By design, it is rather verbose and returns a
    large multiline description
    :return:
    """
    cname = unicode(type(obj).__name__)
    s = [cname, '=' * len(cname), _getdoc(obj),
         '    db format: ' + obj.dbformat,
         '    db name: ' + obj.dbname,
         '    save simulation: ' + str(obj.save_sim),
         '    parallel: ' + type(obj.repeat).__module__.split('.')[-1]]
    return '\n'.join(s)

def setup(obj):
    """
    Describes a spotpy setup using its class name, docstring and parameters
    :param obj: A spotpy compatible model setup
    :return: A describing string
    """
    # Get class name
    cname = unicode(type(obj).__name__)
    # Add doc string
    mdoc = _getdoc(obj).strip('\n').replace('\r', '\n')
    # Get parameters from class
    params = '\n'.join(' - {p}'.format(p=unicode(p)) for p in get_parameters_from_setup(obj))
    parts = [cname, '=' * len(cname), mdoc, 'Parameters:', '-' * 11, params]
    return '\n'.join(parts)

if sys.version_info > (3, 5):

    from pathlib import Path
    import webbrowser

    class rst:
        """
        Creates a reStructuredText description of a sampler or a setup

        Usage:
        >>>description = spotpy.describe.rst(sampler)
        >>>print(description) # Prints the rst source text
        >>># Add additional text section
        >>>description.append('#. One idea' + '\n' + '#. Another one.', title='Ideas', titlelevel=2)
        >>>description.append_image('media/image.png')
        >>>print(description.as_html()) # Create html
        >>>description.show_in_browser()
        """

        caption_characters = '=-#~*+^'

        def __init__(self, setup_or_sampler):
            """
            Creates a reStructuredText description of a sampler or a setup
            :param setup_or_sampler: Either a spotpy.algorithm sampler or a spotpy setup
            """
            if isinstance(setup_or_sampler, _algorithm):
                self.setup = setup_or_sampler.setup
                self.sampler = setup_or_sampler
                self.rst_text = [self._sampler_text()]
            else:
                self.setup = setup_or_sampler
                self.sampler = None
                self.rst_text = []

            if self.setup:
                self.rst_text.append(self._setup_text())

        def append(self, text='', title=None, titlelevel=1):
            """
            Appends additional descriptions in reStructured text to the generated text
            :param text: The rst text to add
            :param title: A title for the text
            :param titlelevel: The level of the section (0->h1.title, 1->h1, 2->h2, etc.)
            :return:
            """
            res = '\n'
            if title:
                res += rst._as_rst_caption(title, titlelevel)
            self.rst_text.append(res + text)

        def append_image(self, imgpath, **kwargs):
            """
            Links an image to the output
            :param imgpath: Path to the image (must be found from the http server)
            :param kwargs: Any keyword with value is translated in rst as `:keyword: value`
                            and added to the image description

            >>>description.append_image('https://img.shields.io/travis/thouska/spotpy/master.svg',
            ...                         target='https://github.com/thouska',
            ...                         width='200px')
            """
            rst = '.. image:: {}'.format(imgpath)
            for k, v in kwargs.items():
                rst += '\n  :{}: {}'.format(k, v)
            rst += '\n'
            self.append(rst)

        def append_math(self, latex):
            """
            Appends a block equation to the output
            :param latex: Latex formula
            """
            rst =  '.. math::\n'
            rst += '  ' + latex + '\n'
            self.append(rst)

        def __str__(self):
            return '\n'.join(self.rst_text)

        @classmethod
        def _as_rst_caption(cls, s, level = int):
            """
            Marks text as a section caption
            :param s: String to be marked as caption
            :param caption_sign: Sign for the caption, eg. =, -, #, ~
            :return: The string as rst caption
            """
            return s + '\n' + cls.caption_characters[level] * len(s) + '\n\n'

        css = """
            body, table, div, p, dl {
                font-family: Lucida Grande, Verdana, Geneva, Arial, sans-serif;
                font-size: 16px;
            }
            li>p {
                margin: 0px;
            }
            /* @group Heading Levels */
            h1.title {
                background-color: #fff;
                color: #0040A0;
                text-align: left;
                font-size: 200%;
                border: solid 2px #1f6992;
            }
            h1 {
             background-color: #1f6992;
             color: #fff;
             padding: .2em .5em;
             font-size: 150%;
            }
            h2 {
             background-color: #cde;
             color: #000;
             padding: .2em .5em;
             border-bottom: solid 2px #1f6992;
             font-size: 120%;
            }

            h3 {
                font-size: 100%;
                border-bottom: solid 2px #0040A0;
            }
            div.line {
                font-family: "Lucida Console", "Lucida Sans Typewriter","DejaVu Sans Mono",monospace;
                font-size: 100%;
            }

            img {
                max-width: 720px;
            }

        """

        def as_html(self, css=None):
            """
            Converts the generated reStructuredText as html5

            :css: A string containing a cascading style sheet. If None, the default css is used
            :return: The html document as string
            """
            if publish_string is None:
                raise NotImplementedError('The docutils package needs to be installed')
            args = {'input_encoding': 'unicode',
                    'output_encoding': 'unicode'}
            res = publish_string(source=str(self),
                                 writer_name='html5',
                                 settings_overrides=args)
            style_idx = res.index('</style>')
            css = css or self.css
            # Include css
            res = res[:style_idx] + css + res[style_idx:]
            return res

        def show_in_browser(self, filename=None, css=None):
            """
            Writes the content as html to disk and opens a browser showing the result

            :param filename: The html filename, if None use <setup class name>.html
            :param css: A style string, if None the default style is used
            """
            html = self.as_html(css).replace('unicode', 'utf-8')
            fn = filename or type(self.setup).__name__ + '.html'
            path = Path(fn).absolute()
            path.write_text(html, encoding='utf-8')
            webbrowser.open_new_tab(path.as_uri())

        def _sampler_text(self):
            """
            Generates the rst for the sampler
            :return:
            """
            obj = self.sampler
            cname = rst._as_rst_caption(type(obj).__name__, 0)
            s = [
                 '- **db format:** ' + obj.dbformat,
                 '- **db name:** ' + obj.dbname,
                 '- **save simulation:** ' + str(obj.save_sim),
                 '- **parallel:** ' + type(obj.repeat).__module__.split('.')[-1],
                 '', ''
                 ]
            return cname + _getdoc(obj).strip('\n') + '\n\n' + '\n'.join(s)

        def _setup_text(self):
            """
            Generates the rst for the setup
            :return:
            """
            # Get class name
            obj = self.setup
            cname = rst._as_rst_caption(type(obj).__name__, 0)
            # Add doc string
            mdoc = _getdoc(obj).strip('\n').replace('\r', '\n') + '\n\n'
            # Get parameters from class
            param_caption = rst._as_rst_caption('Parameters', 1)
            params = '\n'.join('#. **{p.name}:** {p}'.format(p=p) for p in get_parameters_from_setup(obj))
            return cname + mdoc + param_caption + params

<commit_msg>Update describe.py<commit_after># -*- coding: utf-8 -*-
"""
Copyright (c) 2018 by Tobias Houska
This file is part of Statistical Parameter Optimization Tool for Python (SPOTPY).
:author: Philipp Kraft

A collection of helper functions to describe spotpy setups

Usage:

>>> spotpy.describe.sampler(sampler)
>>> spotpy.describe.setup(model)
"""
from __future__ import division, absolute_import, unicode_literals
import sys
from .parameter import get_parameters_from_setup
from .algorithms._algorithm import _algorithm
if sys.version_info[0] >= 3:
    from inspect import getdoc as _getdoc
    unicode = str
else:
    def _getdoc(obj):
        u = obj.__doc__
        try:
            return u'\n'.join(l.strip() for l in u.split(u'\n') if l.strip())
        except UnicodeDecodeError:
            raise AssertionError(
                '{}: Docstring uses unicode but {} misses the line ``from __future__ import unicode_literals``'
                .format(obj, type(obj).__module__)
                )

try:
    from docutils.core import publish_string
except ImportError:
    publish_string = None

def describe(obj):
    """
    Returns a long string description of a sampler with its model
    :param obj: A sampler
    :return: str
    """
    return 'Sampler:\n--------\n{}\n\nModel:\n------\n{}'.format(sampler(obj), setup(obj.setup))

def sampler(obj):
    """
    Returns a string representation of the sampler.
    By design, it is rather verbose and returns a
    large multiline description
    :return:
    """
    cname = unicode(type(obj).__name__)
    s = [cname, '=' * len(cname), _getdoc(obj),
         '    db format: ' + obj.dbformat,
         '    db name: ' + obj.dbname,
         '    save simulation: ' + str(obj.save_sim),
         '    parallel: ' + type(obj.repeat).__module__.split('.')[-1]]
    return '\n'.join(s)

def setup(obj):
    """
    Describes a spotpy setup using its class name, docstring and parameters
    :param obj: A spotpy compatible model setup
    :return: A describing string
    """
    # Get class name
    cname = unicode(type(obj).__name__)
    # Add doc string
    mdoc = _getdoc(obj).strip('\n').replace('\r', '\n')
    # Get parameters from class
    params = '\n'.join(' - {p}'.format(p=unicode(p)) for p in get_parameters_from_setup(obj))
    parts = [cname, '=' * len(cname), mdoc, 'Parameters:', '-' * 11, params]
    return '\n'.join(parts)

if sys.version_info > (3, 5):

    from pathlib import Path
    import webbrowser

    class rst:
        """
        Creates a reStructuredText description of a sampler or a setup

        Usage:
        >>>description = spotpy.describe.rst(sampler)
        >>>print(description) # Prints the rst source text
        >>># Add additional text section
        >>>description.append('#. One idea' + '\n' + '#. Another one.', title='Ideas', titlelevel=2)
        >>>description.append_image('media/image.png')
        >>>print(description.as_html()) # Create html
        >>>description.show_in_browser()
        """

        caption_characters = '=-#~*+^'

        def __init__(self, setup_or_sampler):
            """
            Creates a reStructuredText description of a sampler or a setup
            :param setup_or_sampler: Either a spotpy.algorithm sampler or a spotpy setup
            """
            if isinstance(setup_or_sampler, _algorithm):
                self.setup = setup_or_sampler.setup
                self.sampler = setup_or_sampler
                self.rst_text = [self._sampler_text()]
            else:
                self.setup = setup_or_sampler
                self.sampler = None
                self.rst_text = []

            if self.setup:
                self.rst_text.append(self._setup_text())

        def append(self, text='', title=None, titlelevel=1):
            """
            Appends additional descriptions in reStructured text to the generated text
            :param text: The rst text to add
            :param title: A title for the text
            :param titlelevel: The level of the section (0->h1.title, 1->h1, 2->h2, etc.)
            :return:
            """
            res = '\n'
            if title:
                res += rst._as_rst_caption(title, titlelevel)
            self.rst_text.append(res + text)

        def append_image(self, imgpath, **kwargs):
            """
            Links an image to the output
            :param imgpath: Path to the image (must be found from the http server)
            :param kwargs: Any keyword with value is translated in rst as `:keyword: value`
                            and added to the image description

            >>>description.append_image('https://img.shields.io/travis/thouska/spotpy/master.svg',
            ...                         target='https://github.com/thouska',
            ...                         width='200px')
            """
            rst = '.. image:: {}'.format(imgpath)
            for k, v in kwargs.items():
                rst += '\n  :{}: {}'.format(k, v)
            rst += '\n'
            self.append(rst)

        def append_math(self, latex):
            """
            Appends a block equation to the output
            :param latex: Latex formula
            """
            rst =  '.. math::\n'
            rst += '  ' + latex + '\n'
            self.append(rst)

        def __str__(self):
            return '\n'.join(self.rst_text)

        @classmethod
        def _as_rst_caption(cls, s, level=1):
            """
            Marks text as a section caption
            :param s: String to be marked as caption
            :param level: Caption level 0-6, translates to 0=h1.title, 1=h1, 2=h2, etc.
            :return: The string as rst caption
            """
            return s + '\n' + cls.caption_characters[level] * len(s) + '\n\n'

        css = """
            body, table, div, p, dl {
                font-family: Lucida Grande, Verdana, Geneva, Arial, sans-serif;
                font-size: 16px;
            }
            li>p {
                margin: 0px;
            }
            /* @group Heading Levels */
            h1.title {
                background-color: #fff;
                color: #0040A0;
                text-align: left;
                font-size: 200%;
                border: solid 2px #1f6992;
            }
            h1 {
             background-color: #1f6992;
             color: #fff;
             padding: .2em .5em;
             font-size: 150%;
            }
            h2 {
             background-color: #cde;
             color: #000;
             padding: .2em .5em;
             border-bottom: solid 2px #1f6992;
             font-size: 120%;
            }

            h3 {
                font-size: 100%;
                border-bottom: solid 2px #0040A0;
            }
            div.line {
                font-family: "Lucida Console", "Lucida Sans Typewriter","DejaVu Sans Mono",monospace;
                font-size: 100%;
            }

            img {
                max-width: 720px;
            }

        """

        def as_html(self, css=None):
            """
            Converts the generated reStructuredText as html5

            :css: A string containing a cascading style sheet. If None, the default css is used
            :return: The html document as string
            """
            if publish_string is None:
                raise NotImplementedError('The docutils package needs to be installed')
            args = {'input_encoding': 'unicode',
                    'output_encoding': 'unicode'}
            res = publish_string(source=str(self),
                                 writer_name='html5',
                                 settings_overrides=args)
            style_idx = res.index('</style>')
            css = css or self.css
            # Include css
            res = res[:style_idx] + css + res[style_idx:]
            return res

        def show_in_browser(self, filename=None, css=None):
            """
            Writes the content as html to disk and opens a browser showing the result

            :param filename: The html filename, if None use <setup class name>.html
            :param css: A style string, if None the default style is used
            """
            html = self.as_html(css).replace('unicode', 'utf-8')
            fn = filename or type(self.setup).__name__ + '.html'
            path = Path(fn).absolute()
            path.write_text(html, encoding='utf-8')
            webbrowser.open_new_tab(path.as_uri())

        def _sampler_text(self):
            """
            Generates the rst for the sampler
            :return:
            """
            obj = self.sampler
            cname = rst._as_rst_caption(type(obj).__name__, 0)
            s = [
                 '- **db format:** ' + obj.dbformat,
                 '- **db name:** ' + obj.dbname,
                 '- **save simulation:** ' + str(obj.save_sim),
                 '- **parallel:** ' + type(obj.repeat).__module__.split('.')[-1],
                 '', ''
                 ]
            return cname + _getdoc(obj).strip('\n') + '\n\n' + '\n'.join(s)

        def _setup_text(self):
            """
            Generates the rst for the setup
            :return:
            """
            # Get class name
            obj = self.setup
            cname = rst._as_rst_caption(type(obj).__name__, 0)
            # Add doc string
            mdoc = _getdoc(obj).strip('\n').replace('\r', '\n') + '\n\n'
            # Get parameters from class
            param_caption = rst._as_rst_caption('Parameters', 1)
            params = '\n'.join('#. **{p.name}:** {p}'.format(p=p) for p in get_parameters_from_setup(obj))
            return cname + mdoc + param_caption + params

<|endoftext|>


<commit_before>from mock import patch

from ...testcases import DustyIntegrationTestCase
from ...fixtures import busybox_single_app_bundle_fixture

class TestSyncCLI(DustyIntegrationTestCase):
    def setUp(self):
        super(TestSyncCLI, self).setUp()
        busybox_single_app_bundle_fixture()
        self.run_command('bundles activate busyboxa')
        self.run_command('up')

    def test_sync_repo(self):
        self.exec_in_container('busyboxa', 'rm -rf /repo')
        self.assertFileNotInContainer('busyboxa', '/repo/README.md')
        self.run_command('sync fake-repo')
        self.assertFileContentsInContainer('busyboxa',
                                           '/repo/README.md',
                                           '# fake-repo')
<commit_msg>Clean up after sync integration tests<commit_after>from ...testcases import DustyIntegrationTestCase
from ...fixtures import busybox_single_app_bundle_fixture

class TestSyncCLI(DustyIntegrationTestCase):
    def setUp(self):
        super(TestSyncCLI, self).setUp()
        busybox_single_app_bundle_fixture()
        self.run_command('bundles activate busyboxa')
        self.run_command('up')

    def tearDown(self):
        super(TestSyncCLI, self).tearDown()
        self.run_command('bundles deactivate busyboxa')
        try:
            self.run_command('stop')
        except Exception:
            pass

    def test_sync_repo(self):
        self.exec_in_container('busyboxa', 'rm -rf /repo')
        self.assertFileNotInContainer('busyboxa', '/repo/README.md')
        self.run_command('sync fake-repo')
        self.assertFileContentsInContainer('busyboxa',
                                           '/repo/README.md',
                                           '# fake-repo')
<|endoftext|>


<commit_before>from __future__ import print_function
import os
from os.path import join
from datetime import datetime
from dataactcore.aws.s3UrlHandler import s3UrlHandler
from dataactcore.models.domainModels import TASLookup
from dataactcore.models.stagingModels import AwardFinancial
from dataactcore.config import CONFIG_BROKER
from dataactvalidator.filestreaming.sqlLoader import SQLLoader
from dataactvalidator.filestreaming.schemaLoader import SchemaLoader
from dataactvalidator.filestreaming.loadFile import loadDomainValues
from dataactvalidator.scripts.loadTas import loadTas
from tests.integration.baseTestValidator import BaseTestValidator
import unittest

class FileTypeTests(BaseTestValidator):

    @classmethod
    def setUpClass(cls):
        """Set up class-wide resources."""
        super(FileTypeTests, cls).setUpClass()
        #TODO: refactor into a pytest fixture

        user = cls.userId
        # TODO: get rid of this flag once we're using a tempdb for test fixtures
        force_tas_load = False

        print("Uploading files")
        # Upload needed files to S3
        s3FileNameValid = cls.uploadFile("appropValid.csv", user)
        s3FileNameMixed = cls.uploadFile("appropMixed.csv", user)
        s3FileNameProgramValid = cls.uploadFile("programActivityValid.csv", user)
        s3FileNameProgramMixed = cls.uploadFile("programActivityMixed.csv", user)
        s3FileNameAwardFinValid = cls.uploadFile("awardFinancialValid.csv", user)
        s3FileNameAwardFinMixed = cls.uploadFile("awardFinancialMixed.csv", user)
        s3FileNameAwardValid = cls.uploadFile("awardValid.csv", user)
        s3FileNameAwardMixed = cls.uploadFile("awardMixed.csv", user)
        s3FileNameAwardMixedDelimiter = cls.uploadFile("awardMixedDelimiter.csv", user)
        s3FileNameCrossAwardFin = cls.uploadFile("cross_file_C.csv", user)
        s3FileNameCrossAward = cls.uploadFile("cross_file_D2.csv", user)
        s3FileNameCrossApprop = cls.uploadFile("cross_file_A.csv", user)
        s3FileNameCrossPgmAct = cls.uploadFile("cross_file_B.csv", user)
        s3FileNameAppropValidShortcols = cls.uploadFile("appropValidShortcols.csv", user)
        s3FileNameProgramMixedShortcols = cls.uploadFile("programActivityMixedShortcols.csv", user)
        s3FileNameAwardFinMixedShortcols = cls.uploadFile("awardFinancialMixedShortcols.csv", user)
        s3FileNameAwardValidShortcols = cls.uploadFile("awardValidShortcols.csv", user)

        # Create submissions and get IDs back
        submissionIDs = {}
        for i in range(0, 16):
            if i == 7:
               # Award financial mixed will be second quarter
                submissionIDs[i] = cls.insertSubmission(cls.jobTracker, user, datetime(2015,3,15))
            else:
                submissionIDs[i] = cls.insertSubmission(cls.jobTracker, user)

        # Create jobs
        jobDb = cls.jobTracker
        statusReady = str(jobDb.getJobStatusId("ready"))
        jobTypeCsv = str(jobDb.getJobTypeId("csv_record_validation"))
        jobTypeValidation = str(jobDb.getJobTypeId("validation"))
        jobInfoList = {
            "valid": [statusReady, jobTypeCsv, str(submissionIDs[1]), s3FileNameValid, 1],
            "mixed": [statusReady, jobTypeCsv, str(submissionIDs[2]), s3FileNameMixed, 1],
            "programValid": [statusReady, jobTypeCsv, str(submissionIDs[4]), s3FileNameProgramValid, 2],
            "programMixed": [statusReady, jobTypeCsv, str(submissionIDs[5]), s3FileNameProgramMixed, 2],
            "awardFinValid": [statusReady, jobTypeCsv, str(submissionIDs[6]), s3FileNameAwardFinValid, 3],
            "awardFinMixed": [statusReady, jobTypeCsv, str(submissionIDs[7]), s3FileNameAwardFinMixed, 3],
            "awardValid": [statusReady, jobTypeCsv, str(submissionIDs[8]), s3FileNameAwardValid, 4],
            "awardMixed": [statusReady, jobTypeCsv, str(submissionIDs[9]), s3FileNameAwardMixed, 4],
            "awardMixedDelimiter": [statusReady, jobTypeCsv, str(submissionIDs[10]), s3FileNameAwardMixedDelimiter, 4],
            "crossApprop": [statusReady, jobTypeCsv, str(submissionIDs[11]), s3FileNameCrossApprop, 1],
            "crossPgmAct": [statusReady, jobTypeCsv, str(submissionIDs[11]), s3FileNameCrossPgmAct, 2],
            "crossAwardFin": [statusReady, jobTypeCsv, str(submissionIDs[11]), s3FileNameCrossAwardFin, 3],
            "crossAward": [statusReady, jobTypeCsv, str(submissionIDs[11]), s3FileNameCrossAward, 4],
            "crossFile": [statusReady, jobTypeValidation, str(submissionIDs[11]), None, None],
            "appropValidShortcols": [statusReady, jobTypeCsv, str(submissionIDs[12]), s3FileNameAppropValidShortcols, 1],
            "programMixedShortcols": [statusReady, jobTypeCsv, str(submissionIDs[13]), s3FileNameProgramMixedShortcols, 2],
            "awardFinMixedShortcols": [statusReady, jobTypeCsv, str(submissionIDs[14]), s3FileNameAwardFinMixedShortcols, 3],
            "awardValidShortcols": [statusReady, jobTypeCsv, str(submissionIDs[15]), s3FileNameAwardValidShortcols, 4]
        }

        jobIdDict = {}
        for key in jobInfoList:
            jobInfo = jobInfoList[key]  # Done this way to be compatible with python 2 and 3
            jobInfo.append(jobDb.session)
            job = cls.addJob(*jobInfo)
            jobId = job.job_id
            jobIdDict[key] = jobId
            print("".join([str(key),": ",str(cls.jobTracker.getSubmissionId(jobId)), ", "]), end = "")

        # Load fields and rules
        FileTypeTests.load_definitions(cls.interfaces, force_tas_load)

        cls.jobIdDict = jobIdDict

    @staticmethod
    def load_definitions(interfaces, force_tas_load):
        """Load file definitions."""
        SchemaLoader.loadAllFromPath(join(CONFIG_BROKER["path"],"dataactvalidator","config"))
        SQLLoader.loadSql("sqlRules.csv")
        # Load domain values tables
        loadDomainValues(
            join(CONFIG_BROKER["path"], "dataactvalidator", "config"),
            join(CONFIG_BROKER["path"], os.path.join("tests", "integration", "data"), "sf_133.csv"),
            join(CONFIG_BROKER["path"], os.path.join("tests", "integration", "data"), "program_activity.csv"))
        if (interfaces.validationDb.session.query(TASLookup).count() == 0
                or force_tas_load):
            # TAS table is empty, load it
            loadTas(tasFile=os.path.join(CONFIG_BROKER["path"], "tests", "integration", "data", "all_tas_betc.csv"), dropIdx=False)

    def test_approp_valid(self):
        """Test valid job."""
        jobId = self.jobIdDict["valid"]
        self.passed = self.run_test(
            jobId, 200, "finished", 63, 10, "complete", 0, False)

    def test_approp_valid_shortcol(self):
        """Test valid approp job with short colnames."""
        jobId = self.jobIdDict["appropValidShortcols"]
        self.passed = self.run_test(
            jobId, 200, "finished", 63, 10, "complete", 0, False)

    def test_approp_mixed(self):
        """Test mixed job with some rows failing."""
        jobId = self.jobIdDict["mixed"]
        self.passed = self.run_test(
            jobId, 200, "finished", 8212, 4, "complete", 39, True, 8, 841)

    def test_program_valid(self):
        """Test valid job."""
        jobId = self.jobIdDict["programValid"]
        self.passed = self.run_test(
            jobId, 200, "finished", 63, 10, "complete", 0, False)

    def test_program_mixed(self):
        """Test mixed job with some rows failing."""
        jobId = self.jobIdDict["programMixed"]
        self.passed = self.run_test(
        jobId, 200, "finished", 12058, 4, "complete", 84, True, 27, 10510)

    def test_program_mixed_shortcols(self):
        """Test object class/program activity job with some rows failing & short colnames."""
        jobId = self.jobIdDict["programMixedShortcols"]
        self.passed = self.run_test(
            jobId, 200, "finished", 12058, 4, "complete", 84, True, 27, 10510)

    def test_award_fin_valid(self):
        """Test valid job."""
        jobId = self.jobIdDict["awardFinValid"]
        self.passed = self.run_test(
            jobId, 200, "finished", 63, 11, "complete", 0, False)

    def test_award_fin_mixed(self):
        """Test mixed job with some rows failing."""
        jobId = self.jobIdDict["awardFinMixed"]
        self.passed = self.run_test(
        jobId, 200, "finished", 8000, 7, "complete", 48, True, 29, 9127)

        # Test that whitespace is converted to null
        rowThree = self.interfaces.validationDb.session.query(AwardFinancial).filter(AwardFinancial.parent_award_id == "ZZZZ").filter(AwardFinancial.submission_id == self.interfaces.jobDb.getSubmissionId(jobId)).first()
        self.assertIsNone(rowThree.agency_identifier)
        self.assertIsNone(rowThree.piid)
        # And commas removed for numeric
        rowThirteen = self.interfaces.validationDb.session.query(AwardFinancial).filter(AwardFinancial.parent_award_id == "YYYY").filter(AwardFinancial.submission_id == self.interfaces.jobDb.getSubmissionId(jobId)).first()
        self.assertEqual(rowThirteen.deobligations_recov_by_awa_cpe,26000)

    def test_award_fin_mixed_shortcols(self):
        """Test award financial job with some rows failing & short colnames."""
        jobId = self.jobIdDict["awardFinMixedShortcols"]
        self.passed = self.run_test(
            jobId, 200, "finished", 8000, 7, "complete", 48, True, 31, 11069)

    def test_award_valid(self):
        """Test valid job."""
        jobId = self.jobIdDict["awardValid"]
        self.passed = self.run_test(
            jobId, 200, "finished", 63, 10, "complete", 0, False)

    def test_award_valid_shortcols(self):
        """Test valid award (financial assistance) job with short colnames."""
        jobId = self.jobIdDict["awardValidShortcols"]
        self.passed = self.run_test(
            jobId, 200, "finished", 63, 10, "complete", 0, False)

    def test_award_mixed(self):
        """Test mixed job with some rows failing."""
        jobId = self.jobIdDict["awardMixed"]
        self.passed = self.run_test(
            jobId, 200, "finished", 1139, 8, "complete", 17, True, 24, 2085)

    def test_award_mixed_delimiter(self):
        """Test mixed job with mixed delimiter"""
        jobId = self.jobIdDict["awardMixedDelimiter"]
        self.passed = self.run_test(
            jobId, 400, "invalid", False, False, "header_error", 0, False)

    def test_cross_file(self):
        crossId = self.jobIdDict["crossFile"]
        # Run jobs for A, B, C, and D2, then cross file validation job
        # Note: test files used for cross validation use the short column names
        # as a way to ensure those are handled correctly by the validator
        awardFinResponse = self.validateJob(self.jobIdDict["crossAwardFin"],self.useThreads)
        self.assertEqual(awardFinResponse.status_code, 200, msg=str(awardFinResponse.json))
        awardResponse = self.validateJob(self.jobIdDict["crossAward"],self.useThreads)
        self.assertEqual(awardResponse.status_code, 200, msg=str(awardResponse.json))
        appropResponse = self.validateJob(self.jobIdDict["crossApprop"], self.useThreads)
        self.assertEqual(appropResponse.status_code, 200, msg=str(appropResponse.json))
        pgmActResponse = self.validateJob(self.jobIdDict["crossPgmAct"], self.useThreads)
        self.assertEqual(pgmActResponse.status_code, 200, msg=str(pgmActResponse.json))
        crossFileResponse = self.validateJob(crossId, self.useThreads)
        self.assertEqual(crossFileResponse.status_code, 200, msg=str(crossFileResponse.json))

        # Check number of cross file validation errors in DB for this job
        self.assertEqual(self.interfaces.errorDb.checkNumberOfErrorsByJobId(crossId, self.interfaces.validationDb, "fatal"), 0)
        self.assertEqual(self.interfaces.errorDb.checkNumberOfErrorsByJobId(crossId, self.interfaces.validationDb, "warning"), 5)
        # Check cross file job complete
        self.waitOnJob(self.interfaces.jobDb, crossId, "finished", self.useThreads)
        # Check that cross file validation report exists and is the right size
        jobTracker = self.interfaces.jobDb

        submissionId = jobTracker.getSubmissionId(crossId)
        sizePathPairs = [
            (89, self.interfaces.errorDb.getCrossReportName(submissionId, "appropriations", "program_activity")),
            (89, self.interfaces.errorDb.getCrossReportName(submissionId, "award_financial", "award")),
            (1329, self.interfaces.errorDb.getCrossWarningReportName(submissionId, "appropriations", "program_activity")),
            (424, self.interfaces.errorDb.getCrossWarningReportName(submissionId, "award_financial", "award")),
        ]

        for size, path in sizePathPairs:
            if self.local:
                self.assertFileSizeAppxy(size, path)
            else:
                self.assertGreater(
                    s3UrlHandler.getFileSize("errors/" + path), size - 5)
                self.assertLess(
                    s3UrlHandler.getFileSize("errors/" + path), size + 5)

if __name__ == '__main__':
    unittest.main()
<commit_msg>Updating fileTypeTests for accommodate for changes to rule severities for some rules<commit_after>from __future__ import print_function
import os
from os.path import join
from datetime import datetime
from dataactcore.aws.s3UrlHandler import s3UrlHandler
from dataactcore.models.domainModels import TASLookup
from dataactcore.models.stagingModels import AwardFinancial
from dataactcore.config import CONFIG_BROKER
from dataactvalidator.filestreaming.sqlLoader import SQLLoader
from dataactvalidator.filestreaming.schemaLoader import SchemaLoader
from dataactvalidator.filestreaming.loadFile import loadDomainValues
from dataactvalidator.scripts.loadTas import loadTas
from tests.integration.baseTestValidator import BaseTestValidator
import unittest

class FileTypeTests(BaseTestValidator):

    @classmethod
    def setUpClass(cls):
        """Set up class-wide resources."""
        super(FileTypeTests, cls).setUpClass()
        #TODO: refactor into a pytest fixture

        user = cls.userId
        # TODO: get rid of this flag once we're using a tempdb for test fixtures
        force_tas_load = False

        print("Uploading files")
        # Upload needed files to S3
        s3FileNameValid = cls.uploadFile("appropValid.csv", user)
        s3FileNameMixed = cls.uploadFile("appropMixed.csv", user)
        s3FileNameProgramValid = cls.uploadFile("programActivityValid.csv", user)
        s3FileNameProgramMixed = cls.uploadFile("programActivityMixed.csv", user)
        s3FileNameAwardFinValid = cls.uploadFile("awardFinancialValid.csv", user)
        s3FileNameAwardFinMixed = cls.uploadFile("awardFinancialMixed.csv", user)
        s3FileNameAwardValid = cls.uploadFile("awardValid.csv", user)
        s3FileNameAwardMixed = cls.uploadFile("awardMixed.csv", user)
        s3FileNameAwardMixedDelimiter = cls.uploadFile("awardMixedDelimiter.csv", user)
        s3FileNameCrossAwardFin = cls.uploadFile("cross_file_C.csv", user)
        s3FileNameCrossAward = cls.uploadFile("cross_file_D2.csv", user)
        s3FileNameCrossApprop = cls.uploadFile("cross_file_A.csv", user)
        s3FileNameCrossPgmAct = cls.uploadFile("cross_file_B.csv", user)
        s3FileNameAppropValidShortcols = cls.uploadFile("appropValidShortcols.csv", user)
        s3FileNameProgramMixedShortcols = cls.uploadFile("programActivityMixedShortcols.csv", user)
        s3FileNameAwardFinMixedShortcols = cls.uploadFile("awardFinancialMixedShortcols.csv", user)
        s3FileNameAwardValidShortcols = cls.uploadFile("awardValidShortcols.csv", user)

        # Create submissions and get IDs back
        submissionIDs = {}
        for i in range(0, 16):
            if i == 7:
               # Award financial mixed will be second quarter
                submissionIDs[i] = cls.insertSubmission(cls.jobTracker, user, datetime(2015,3,15))
            else:
                submissionIDs[i] = cls.insertSubmission(cls.jobTracker, user)

        # Create jobs
        jobDb = cls.jobTracker
        statusReady = str(jobDb.getJobStatusId("ready"))
        jobTypeCsv = str(jobDb.getJobTypeId("csv_record_validation"))
        jobTypeValidation = str(jobDb.getJobTypeId("validation"))
        jobInfoList = {
            "valid": [statusReady, jobTypeCsv, str(submissionIDs[1]), s3FileNameValid, 1],
            "mixed": [statusReady, jobTypeCsv, str(submissionIDs[2]), s3FileNameMixed, 1],
            "programValid": [statusReady, jobTypeCsv, str(submissionIDs[4]), s3FileNameProgramValid, 2],
            "programMixed": [statusReady, jobTypeCsv, str(submissionIDs[5]), s3FileNameProgramMixed, 2],
            "awardFinValid": [statusReady, jobTypeCsv, str(submissionIDs[6]), s3FileNameAwardFinValid, 3],
            "awardFinMixed": [statusReady, jobTypeCsv, str(submissionIDs[7]), s3FileNameAwardFinMixed, 3],
            "awardValid": [statusReady, jobTypeCsv, str(submissionIDs[8]), s3FileNameAwardValid, 4],
            "awardMixed": [statusReady, jobTypeCsv, str(submissionIDs[9]), s3FileNameAwardMixed, 4],
            "awardMixedDelimiter": [statusReady, jobTypeCsv, str(submissionIDs[10]), s3FileNameAwardMixedDelimiter, 4],
            "crossApprop": [statusReady, jobTypeCsv, str(submissionIDs[11]), s3FileNameCrossApprop, 1],
            "crossPgmAct": [statusReady, jobTypeCsv, str(submissionIDs[11]), s3FileNameCrossPgmAct, 2],
            "crossAwardFin": [statusReady, jobTypeCsv, str(submissionIDs[11]), s3FileNameCrossAwardFin, 3],
            "crossAward": [statusReady, jobTypeCsv, str(submissionIDs[11]), s3FileNameCrossAward, 4],
            "crossFile": [statusReady, jobTypeValidation, str(submissionIDs[11]), None, None],
            "appropValidShortcols": [statusReady, jobTypeCsv, str(submissionIDs[12]), s3FileNameAppropValidShortcols, 1],
            "programMixedShortcols": [statusReady, jobTypeCsv, str(submissionIDs[13]), s3FileNameProgramMixedShortcols, 2],
            "awardFinMixedShortcols": [statusReady, jobTypeCsv, str(submissionIDs[14]), s3FileNameAwardFinMixedShortcols, 3],
            "awardValidShortcols": [statusReady, jobTypeCsv, str(submissionIDs[15]), s3FileNameAwardValidShortcols, 4]
        }

        jobIdDict = {}
        for key in jobInfoList:
            jobInfo = jobInfoList[key]  # Done this way to be compatible with python 2 and 3
            jobInfo.append(jobDb.session)
            job = cls.addJob(*jobInfo)
            jobId = job.job_id
            jobIdDict[key] = jobId
            print("".join([str(key),": ",str(cls.jobTracker.getSubmissionId(jobId)), ", "]), end = "")

        # Load fields and rules
        FileTypeTests.load_definitions(cls.interfaces, force_tas_load)

        cls.jobIdDict = jobIdDict

    @staticmethod
    def load_definitions(interfaces, force_tas_load):
        """Load file definitions."""
        SchemaLoader.loadAllFromPath(join(CONFIG_BROKER["path"],"dataactvalidator","config"))
        SQLLoader.loadSql("sqlRules.csv")
        # Load domain values tables
        loadDomainValues(
            join(CONFIG_BROKER["path"], "dataactvalidator", "config"),
            join(CONFIG_BROKER["path"], os.path.join("tests", "integration", "data"), "sf_133.csv"),
            join(CONFIG_BROKER["path"], os.path.join("tests", "integration", "data"), "program_activity.csv"))
        if (interfaces.validationDb.session.query(TASLookup).count() == 0
                or force_tas_load):
            # TAS table is empty, load it
            loadTas(tasFile=os.path.join(CONFIG_BROKER["path"], "tests", "integration", "data", "all_tas_betc.csv"), dropIdx=False)

    def test_approp_valid(self):
        """Test valid job."""
        jobId = self.jobIdDict["valid"]
        self.passed = self.run_test(
            jobId, 200, "finished", 63, 10, "complete", 0, False)

    def test_approp_valid_shortcol(self):
        """Test valid approp job with short colnames."""
        jobId = self.jobIdDict["appropValidShortcols"]
        self.passed = self.run_test(
            jobId, 200, "finished", 63, 10, "complete", 0, False)

    def test_approp_mixed(self):
        """Test mixed job with some rows failing."""
        jobId = self.jobIdDict["mixed"]
        self.passed = self.run_test(
            jobId, 200, "finished", 8212, 4, "complete", 39, True, 8, 841)

    def test_program_valid(self):
        """Test valid job."""
        jobId = self.jobIdDict["programValid"]
        self.passed = self.run_test(
            jobId, 200, "finished", 63, 10, "complete", 0, False)

    def test_program_mixed(self):
        """Test mixed job with some rows failing."""
        jobId = self.jobIdDict["programMixed"]
        self.passed = self.run_test(
        jobId, 200, "finished", 11390, 4, "complete", 81, True, 29, 10853)

    def test_program_mixed_shortcols(self):
        """Test object class/program activity job with some rows failing & short colnames."""
        jobId = self.jobIdDict["programMixedShortcols"]
        self.passed = self.run_test(
            jobId, 200, "finished", 11390, 4, "complete", 81, True, 29, 10853)

    def test_award_fin_valid(self):
        """Test valid job."""
        jobId = self.jobIdDict["awardFinValid"]
        self.passed = self.run_test(
            jobId, 200, "finished", 63, 11, "complete", 0, False)

    def test_award_fin_mixed(self):
        """Test mixed job with some rows failing."""
        jobId = self.jobIdDict["awardFinMixed"]
        self.passed = self.run_test(
        jobId, 200, "finished", 7332, 7, "complete", 45, True, 32, 9795)

        # Test that whitespace is converted to null
        rowThree = self.interfaces.validationDb.session.query(AwardFinancial).filter(AwardFinancial.parent_award_id == "ZZZZ").filter(AwardFinancial.submission_id == self.interfaces.jobDb.getSubmissionId(jobId)).first()
        self.assertIsNone(rowThree.agency_identifier)
        self.assertIsNone(rowThree.piid)
        # And commas removed for numeric
        rowThirteen = self.interfaces.validationDb.session.query(AwardFinancial).filter(AwardFinancial.parent_award_id == "YYYY").filter(AwardFinancial.submission_id == self.interfaces.jobDb.getSubmissionId(jobId)).first()
        self.assertEqual(rowThirteen.deobligations_recov_by_awa_cpe,26000)

    def test_award_fin_mixed_shortcols(self):
        """Test award financial job with some rows failing & short colnames."""
        jobId = self.jobIdDict["awardFinMixedShortcols"]
        self.passed = self.run_test(
            jobId, 200, "finished", 7332, 7, "complete", 45, True, 34, 11737)

    def test_award_valid(self):
        """Test valid job."""
        jobId = self.jobIdDict["awardValid"]
        self.passed = self.run_test(
            jobId, 200, "finished", 63, 10, "complete", 0, False)

    def test_award_valid_shortcols(self):
        """Test valid award (financial assistance) job with short colnames."""
        jobId = self.jobIdDict["awardValidShortcols"]
        self.passed = self.run_test(
            jobId, 200, "finished", 63, 10, "complete", 0, False)

    def test_award_mixed(self):
        """Test mixed job with some rows failing."""
        jobId = self.jobIdDict["awardMixed"]
        self.passed = self.run_test(
            jobId, 200, "finished", 1139, 8, "complete", 17, True, 24, 2085)

    def test_award_mixed_delimiter(self):
        """Test mixed job with mixed delimiter"""
        jobId = self.jobIdDict["awardMixedDelimiter"]
        self.passed = self.run_test(
            jobId, 400, "invalid", False, False, "header_error", 0, False)

    def test_cross_file(self):
        crossId = self.jobIdDict["crossFile"]
        # Run jobs for A, B, C, and D2, then cross file validation job
        # Note: test files used for cross validation use the short column names
        # as a way to ensure those are handled correctly by the validator
        awardFinResponse = self.validateJob(self.jobIdDict["crossAwardFin"],self.useThreads)
        self.assertEqual(awardFinResponse.status_code, 200, msg=str(awardFinResponse.json))
        awardResponse = self.validateJob(self.jobIdDict["crossAward"],self.useThreads)
        self.assertEqual(awardResponse.status_code, 200, msg=str(awardResponse.json))
        appropResponse = self.validateJob(self.jobIdDict["crossApprop"], self.useThreads)
        self.assertEqual(appropResponse.status_code, 200, msg=str(appropResponse.json))
        pgmActResponse = self.validateJob(self.jobIdDict["crossPgmAct"], self.useThreads)
        self.assertEqual(pgmActResponse.status_code, 200, msg=str(pgmActResponse.json))
        crossFileResponse = self.validateJob(crossId, self.useThreads)
        self.assertEqual(crossFileResponse.status_code, 200, msg=str(crossFileResponse.json))

        # Check number of cross file validation errors in DB for this job
        self.assertEqual(self.interfaces.errorDb.checkNumberOfErrorsByJobId(crossId, self.interfaces.validationDb, "fatal"), 0)
        self.assertEqual(self.interfaces.errorDb.checkNumberOfErrorsByJobId(crossId, self.interfaces.validationDb, "warning"), 5)
        # Check cross file job complete
        self.waitOnJob(self.interfaces.jobDb, crossId, "finished", self.useThreads)
        # Check that cross file validation report exists and is the right size
        jobTracker = self.interfaces.jobDb

        submissionId = jobTracker.getSubmissionId(crossId)
        sizePathPairs = [
            (89, self.interfaces.errorDb.getCrossReportName(submissionId, "appropriations", "program_activity")),
            (89, self.interfaces.errorDb.getCrossReportName(submissionId, "award_financial", "award")),
            (1329, self.interfaces.errorDb.getCrossWarningReportName(submissionId, "appropriations", "program_activity")),
            (424, self.interfaces.errorDb.getCrossWarningReportName(submissionId, "award_financial", "award")),
        ]

        for size, path in sizePathPairs:
            if self.local:
                self.assertFileSizeAppxy(size, path)
            else:
                self.assertGreater(
                    s3UrlHandler.getFileSize("errors/" + path), size - 5)
                self.assertLess(
                    s3UrlHandler.getFileSize("errors/" + path), size + 5)

if __name__ == '__main__':
    unittest.main()
<|endoftext|>


<commit_before>"""Define functions to send emails."""

import os

from pyramid.renderers import render
from pyramid_mailer import get_mailer
from pyramid_mailer.message import Attachment
from pyramid_mailer.message import Message

from jinja2 import Environment, Template, TemplateNotFound, FileSystemLoader
import html2text

from spynl.main.utils import get_logger, get_settings

from spynl.main.exceptions import EmailTemplateNotFound, EmailRecipientNotGiven
from .locale import TemplateTranslations, SpynlTranslationString

DEFAULT_HTML_TEMPLATE = '''
    <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
                        "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">

        <meta http-equiv="Content-Type" content="text/html; charset=UTF-8" />
        <title>{{subject}}</title>
        <meta name="viewport" content="width=device-width, initial-scale=1.0"/>

            {{content}}

    '''

def _sendmail(request, recipient, subject, plain_body, html_body=None,
              sender=None, attachments=None, fail_silently=True, mailer=None,
              reply_to=None):
    """
    Send a mail using the pyramid mailer. This function also makes sure
    that if Spynl is not in a production environment, mail is sent to a dummy
    email address.

    :param Request request: the original request
    :param string recipient: addressee
    :param string subject: subject of mail
    :param string plain_body: content of mail
    :param string html_body: html content of mail
    :param string sender: senders address (optional,
                          default: no_reply@<spynl.domain>)
    :param [Attachment] attachments: attachments (optional)
    :param bool fail_silently: keep quiet when connection errors happen
    :param pyramid_mailer.Mailer mailer: mailer object (optional)
    """
    settings = get_settings()
    logger = get_logger()

    if mailer is None:
        mailer = get_mailer(request)

    if not sender:
        sender = settings.get('mail.sender')
        if not sender:
            domain = settings.get('spynl.domain', 'example.com')
            sender = 'no_reply@{}'.format(domain)

    sender = str(sender)
    recipients = None
    # Recipient could be given as None, in which case we'll use the dummy
    # recipient.
    if recipient:
        recipients = [str(recipient)]
    elif not fail_silently:
        raise EmailRecipientNotGiven()
    elif settings.get('mail.dummy_recipient'):
        recipients = [settings.get('mail.dummy_recipient')]
        logger.info("I will send this email to %s instead of %s.",
                    settings.get('mail.dummy_recipient'), recipient)
    subject = str(subject).rstrip()

    # set Reply-To header if needed:
    extra_headers = {}
    if reply_to:
        extra_headers['Reply-To'] = reply_to
    message = Message(sender=sender, recipients=recipients, subject=subject,
                      body=plain_body, html=html_body,
                      extra_headers=extra_headers)
    if attachments:
        for attachment in attachments:
            message.attach(attachment)

    try:
        message.validate()
        logger.info('Sending email titled "%s" to %s from %s',
                    subject, recipients, sender)
        # Always set fail_silently to false, so we can log the
        # exception
        mailer.send_immediately(message, fail_silently=False)
    except Exception as e:
        if fail_silently:
            logger.exception(e)
            return False
        else:
            raise e

    return True

def send_template_email(request, recipient, template_string=None,
                        template_file=None, plugin=None, replacements={},
                        subject='', fail_silently=True, mailer=None,
                        attachments=None, sender=None, reply_to=None):
    """
    Send email using a html template.

    The email's content (which can contain HTML code) is defined by a Jinja
    template. This content template can be given either by filename
    <template_file> (absolute path) or directly in string form
    <template_string>. In both cases, the email content gets wrapped by a base
    template which defines a consistent layout. The base template to use is
    defined by a string type setting:
        `base_email_template: absolute path of template
    If the "base template" cannot be loaded the <DEFAULT_HTML_TEMPLATE> is
    used.
    Replacements can be send in to customise the email content.
    If no html or plain text was constructed in the end, replacements are being
    used to construct a basic text to be send as an email.

    In case of a jinja2 template the following assignments are attempted to be
    loaded from the template:
        `default_subject`: will be used if no subject was passed
    """
    if (template_string is None and template_file is None) or \
            (template_string is not None and template_file is not None):
        raise Exception('One of <template_string> or <template_file> must be '
                        'given.')
    text_body = ''
    if subject:
        replacements['subject'] = subject

    for key, value in replacements.items():
        if isinstance(value, SpynlTranslationString):
            replacements[key] = value.translate()

    if template_file is not None:
        try:
            replacements['content'] = render(template_file, replacements,
                                             request=request)
        except TemplateNotFound:
            raise EmailTemplateNotFound(template_file)

        if not subject:  # update with possible subject defined in template
            subject = value_from_template(template_file, 'default_subject',
                                          request, replacements, plugin)
            replacements.update(subject=subject)
    else:
        replacements['content'] = Template(template_string).render(
            **replacements)

    # Render the base template with the content of the given template
    base_template = get_settings().get('base_email_template')
    if base_template is not None:
        html_body = render(base_template, replacements, request=request)
    else:
        html_body = Template(DEFAULT_HTML_TEMPLATE).render(**replacements)
    html_body = html_body.replace('\n', '')

    text_maker = html2text.HTML2Text()
    text_maker.ignore_images = True
    text_body = text_maker.handle(html_body)

    text_body = Attachment(data=text_body, transfer_encoding="base64",
                           content_type="text/plain; charset=UTF-8",
                           disposition='inline')
    html_body = Attachment(data=html_body, transfer_encoding="base64",
                           content_type="text/html; charset=UTF-8",
                           disposition='inline')

    if not text_body and not html_body:
        str_replacements = '\n'.join(['{}: {}'.format(key, value)
                                      for key, value in replacements.items()])
        text_body = 'Your account has been changed.\n' + str_replacements
        template = Template(DEFAULT_HTML_TEMPLATE)
        replacements.update(subject=subject, content=text_body)
        html_body = template.render(**replacements)
        get_logger(__name__).error(
            'Body was not found in body_string or email template: %s',
            template)

    return _sendmail(request, recipient, subject, text_body, html_body,
                     fail_silently=fail_silently, mailer=mailer,
                     attachments=attachments, sender=sender, reply_to=reply_to)

def value_from_template(template_file, var, request, replacements,
                        plugin=None):
    """
    Return the value of given <var> defined inside the <template_file>.

    Set the static_url filter for templates to construct urls for static files.
    If plugin was given, install plugin's translations to the environment.
    """
    path, filename = os.path.split(template_file)
    env = Environment(loader=FileSystemLoader(path),
                      extensions=['jinja2.ext.i18n'])
    env.filters['static_url'] = request.static_url
    if plugin is not None:
        translations = TemplateTranslations(plugin)
        env.install_gettext_translations(translations)

    template = env.get_template(filename)
    template_module = template.make_module(vars=replacements)
    return getattr(template_module, var, '')
<commit_msg>fix adding newlines in links in text version of email<commit_after>"""Define functions to send emails."""

import os

from pyramid.renderers import render
from pyramid_mailer import get_mailer
from pyramid_mailer.message import Attachment
from pyramid_mailer.message import Message

from jinja2 import Environment, Template, TemplateNotFound, FileSystemLoader
import html2text

from spynl.main.utils import get_logger, get_settings

from spynl.main.exceptions import EmailTemplateNotFound, EmailRecipientNotGiven
from .locale import TemplateTranslations, SpynlTranslationString

DEFAULT_HTML_TEMPLATE = '''
    <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
                        "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">

        <meta http-equiv="Content-Type" content="text/html; charset=UTF-8" />
        <title>{{subject}}</title>
        <meta name="viewport" content="width=device-width, initial-scale=1.0"/>

            {{content}}

    '''

def _sendmail(request, recipient, subject, plain_body, html_body=None,
              sender=None, attachments=None, fail_silently=True, mailer=None,
              reply_to=None):
    """
    Send a mail using the pyramid mailer. This function also makes sure
    that if Spynl is not in a production environment, mail is sent to a dummy
    email address.

    :param Request request: the original request
    :param string recipient: addressee
    :param string subject: subject of mail
    :param string plain_body: content of mail
    :param string html_body: html content of mail
    :param string sender: senders address (optional,
                          default: no_reply@<spynl.domain>)
    :param [Attachment] attachments: attachments (optional)
    :param bool fail_silently: keep quiet when connection errors happen
    :param pyramid_mailer.Mailer mailer: mailer object (optional)
    """
    settings = get_settings()
    logger = get_logger()

    if mailer is None:
        mailer = get_mailer(request)

    if not sender:
        sender = settings.get('mail.sender')
        if not sender:
            domain = settings.get('spynl.domain', 'example.com')
            sender = 'no_reply@{}'.format(domain)

    sender = str(sender)
    recipients = None
    # Recipient could be given as None, in which case we'll use the dummy
    # recipient.
    if recipient:
        recipients = [str(recipient)]
    elif not fail_silently:
        raise EmailRecipientNotGiven()
    elif settings.get('mail.dummy_recipient'):
        recipients = [settings.get('mail.dummy_recipient')]
        logger.info("I will send this email to %s instead of %s.",
                    settings.get('mail.dummy_recipient'), recipient)
    subject = str(subject).rstrip()

    # set Reply-To header if needed:
    extra_headers = {}
    if reply_to:
        extra_headers['Reply-To'] = reply_to
    message = Message(sender=sender, recipients=recipients, subject=subject,
                      body=plain_body, html=html_body,
                      extra_headers=extra_headers)
    if attachments:
        for attachment in attachments:
            message.attach(attachment)

    try:
        message.validate()
        logger.info('Sending email titled "%s" to %s from %s',
                    subject, recipients, sender)
        # Always set fail_silently to false, so we can log the
        # exception
        mailer.send_immediately(message, fail_silently=False)
    except Exception as e:
        if fail_silently:
            logger.exception(e)
            return False
        else:
            raise e

    return True

def send_template_email(request, recipient, template_string=None,
                        template_file=None, plugin=None, replacements={},
                        subject='', fail_silently=True, mailer=None,
                        attachments=None, sender=None, reply_to=None):
    """
    Send email using a html template.

    The email's content (which can contain HTML code) is defined by a Jinja
    template. This content template can be given either by filename
    <template_file> (absolute path) or directly in string form
    <template_string>. In both cases, the email content gets wrapped by a base
    template which defines a consistent layout. The base template to use is
    defined by a string type setting:
        `base_email_template: absolute path of template
    If the "base template" cannot be loaded the <DEFAULT_HTML_TEMPLATE> is
    used.
    Replacements can be send in to customise the email content.
    If no html or plain text was constructed in the end, replacements are being
    used to construct a basic text to be send as an email.

    In case of a jinja2 template the following assignments are attempted to be
    loaded from the template:
        `default_subject`: will be used if no subject was passed
    """
    if (template_string is None and template_file is None) or \
            (template_string is not None and template_file is not None):
        raise Exception('One of <template_string> or <template_file> must be '
                        'given.')
    text_body = ''
    if subject:
        replacements['subject'] = subject

    for key, value in replacements.items():
        if isinstance(value, SpynlTranslationString):
            replacements[key] = value.translate()

    if template_file is not None:
        try:
            replacements['content'] = render(template_file, replacements,
                                             request=request)
        except TemplateNotFound:
            raise EmailTemplateNotFound(template_file)

        if not subject:  # update with possible subject defined in template
            subject = value_from_template(template_file, 'default_subject',
                                          request, replacements, plugin)
            replacements.update(subject=subject)
    else:
        replacements['content'] = Template(template_string).render(
            **replacements)

    # Render the base template with the content of the given template
    base_template = get_settings().get('base_email_template')
    if base_template is not None:
        html_body = render(base_template, replacements, request=request)
    else:
        html_body = Template(DEFAULT_HTML_TEMPLATE).render(**replacements)
    html_body = html_body.replace('\n', '')

    text_maker = html2text.HTML2Text()
    text_maker.ignore_images = True
    text_maker.wrap_links = False
    text_body = text_maker.handle(html_body)

    text_body = Attachment(data=text_body, transfer_encoding="base64",
                           content_type="text/plain; charset=UTF-8",
                           disposition='inline')
    html_body = Attachment(data=html_body, transfer_encoding="base64",
                           content_type="text/html; charset=UTF-8",
                           disposition='inline')

    if not text_body and not html_body:
        str_replacements = '\n'.join(['{}: {}'.format(key, value)
                                      for key, value in replacements.items()])
        text_body = 'Your account has been changed.\n' + str_replacements
        template = Template(DEFAULT_HTML_TEMPLATE)
        replacements.update(subject=subject, content=text_body)
        html_body = template.render(**replacements)
        get_logger(__name__).error(
            'Body was not found in body_string or email template: %s',
            template)

    return _sendmail(request, recipient, subject, text_body, html_body,
                     fail_silently=fail_silently, mailer=mailer,
                     attachments=attachments, sender=sender, reply_to=reply_to)

def value_from_template(template_file, var, request, replacements,
                        plugin=None):
    """
    Return the value of given <var> defined inside the <template_file>.

    Set the static_url filter for templates to construct urls for static files.
    If plugin was given, install plugin's translations to the environment.
    """
    path, filename = os.path.split(template_file)
    env = Environment(loader=FileSystemLoader(path),
                      extensions=['jinja2.ext.i18n'])
    env.filters['static_url'] = request.static_url
    if plugin is not None:
        translations = TemplateTranslations(plugin)
        env.install_gettext_translations(translations)

    template = env.get_template(filename)
    template_module = template.make_module(vars=replacements)
    return getattr(template_module, var, '')
<|endoftext|>