<commit_before>from flask import Flask, request
from celery.local import PromiseProxy
from .config import Config

class MsBot:
    def __init__(self, host=None, port=None, debug=None):
        self.processes = []
        config = Config()
        self.host = config.get_config(host, 'HOST', root='flask')
        self.port = config.get_config(port, 'PORT', root='flask')
        self.debug = config.get_config(debug, 'DEBUG', root='flask')

        self.app = Flask(__name__)

        @self.app.route('/api/messages', methods=['POST'])
        def message_post():
            # TODO: Confirm that it is from microsoft

            json_message = request.get_json()

            self.app.logger.info('message.headers: {}'.format(request.headers))
            self.app.logger.info('message.body: {}'.format(json_message))

            for process in self.processes:
                if isinstance(process, PromiseProxy):
                    self.app.logger.info('Processing task {} asynchronously.'.format(type(process).__name__))
                    process.delay(json_message)
                elif callable(process):
                    self.app.logger.info('Processing task {} synchronously.'.format(process.__name__))
                    process(json_message)
            return "Success"

    def add_process(self, process):
        self.processes.append(process)

    def run(self):
        self.app.run(host=self.host, port=self.port, debug=self.debug)
<commit_msg>Checking header information<commit_after>from flask import Flask, request
from celery.local import PromiseProxy
from .config import Config
import requests
import json
from jwt.algorithms import RSAAlgorithm
import jwt

class MsBot:
    def __init__(self, host=None, port=None, debug=None):
        self.processes = []
        config = Config()
        self.host = config.get_config(host, 'HOST', root='flask')
        self.port = config.get_config(port, 'PORT', root='flask')
        self.debug = config.get_config(debug, 'DEBUG', root='flask')

        self.app = Flask(__name__)

        @self.app.route('/api/messages', methods=['POST'])
        def message_post():
            self.verfiy_token(request, self.app.logger)

            json_message = request.get_json()

            self.app.logger.info('message.headers: {}'.format(request.headers))
            self.app.logger.info('message.body: {}'.format(json_message))

            for process in self.processes:
                if isinstance(process, PromiseProxy):
                    self.app.logger.info('Processing task {} asynchronously.'.format(type(process).__name__))
                    process.delay(json_message)
                elif callable(process):
                    self.app.logger.info('Processing task {} synchronously.'.format(process.__name__))
                    process(json_message)
            return "Success"

        def verify_token(self, request, logger, app_client_id=None):
            print(request.headers)
            print(request.get_json())

            authorization_header = request.headers['authorisation']
            token = authorization_header[7:]
            authorization_scheme = authorization_header[:6]

            config = Config()
            audience = config.get_config(app_client_id, 'APP_CLIENT_ID')

            # 1. The token was sent in the HTTP Authorization header with 'Bearer' scheme
            if authorization_scheme == "Bearer":
                logger.warning('The token was not sent in the http authorisation header with the Bearer scheme.')
                return False

            # 2. The token is valid JSON that conforms to the JWT standard (see references)
            # 4. The token contains an audience claim with a value equivalent to your bot's Microsoft App ID.
            # 5. The token has not yet expired. Industry-standard clock-skew is 5 minutes.
            # 6. The token has a valid cryptographic signature with a key listed in the OpenId keys document retrieved in step 1, above.

            openid_metadata_url = "https://login.botframework.com/v1/.well-known/openidconfiguration"
            openid_metadata = requests.get(openid_metadata_url)
            print(openid_metadata.text)

            valid_signing_keys_url = openid_metadata.json()["jwks_uri"]
            valid_signing_keys = requests.get(valid_signing_keys_url)
            valid_signing_keys = valid_signing_keys.json()

            for dict_key in valid_signing_keys['keys']:
                # if dict_key['kid'] ==

                key = json.dumps(dict_key)

                algo = RSAAlgorithm('SHA256')
                public_key = algo.from_jwk(key)

                try:
                    decoded_jwt = jwt.decode(token, public_key, algorithms=['RS256'], audience=audience)
                    print(decoded_jwt)
                except jwt.exceptions.InvalidTokenError as e:
                    print(e)

                    # 3. The token contains an issuer claim with value of https://api.botframework.com

    def add_process(self, process):
        self.processes.append(process)

    def run(self):
        self.app.run(host=self.host, port=self.port, debug=self.debug)

<|endoftext|>


<commit_before>import numpy
import pytest

import chainer
import chainerx
import chainerx.testing

from chainerx_tests import array_utils
from chainerx_tests import dtype_utils
from chainerx_tests import op_utils

@op_utils.op_test(['native:0', 'cuda:0'])
@chainer.testing.parameterize_pytest('a_shape,b_shape', [
    ((), ()),
    ((), (2, 3)),
    ((2, 0), (0, 3)),
    ((0, 0), (0, 0)),
    ((2, 3), (3, 4)),
    ((4, 5, 2), (3, 2, 5)),
    ((2, 3, 4, 4), (3, 4, 2)),
    ((2, 2, 3, 1), (2, 1, 3, 1,  4)),
    ((2, 4, 3), (1, 2, 3, 2))

    # TODO(niboshi): Add test cases for more than 2 ndim
])
@chainer.testing.parameterize_pytest(
    'in_dtypes,chx_expected_dtype', dtype_utils.result_dtypes_two_arrays)
@chainer.testing.parameterize_pytest('is_module', [True, False])
class TestDot(op_utils.NumpyOpTest):

    def setup(self):
        device = chainerx.get_default_device()
        a_dtype, b_dtype = self.in_dtypes
        a_kind = numpy.dtype(a_dtype).kind
        b_kind = numpy.dtype(b_dtype).kind
        # TODO(beam2d): Remove the skip after supporting non-float dot on CUDA
        if device.name == 'cuda:0' and (a_kind != 'f' and b_kind != 'f'):
            pytest.skip('non-float dot is not supported on CUDA')

        # Skip backward/double-backward tests for int dtypes
        if a_kind != 'f' or b_kind != 'f':
            self.skip_backward_test = True
            self.skip_double_backward_test = True
        # Skip backward/double-backward tests if the output will be
        # disconnected.
        # TODO(niboshi): Remove this skip condition after enabling backward()
        # for such cases.
        if self.a_shape and self.a_shape[-1] == 0:
            self.skip_backward_test = True
            self.skip_double_backward_test = True

        if a_dtype == 'float16' or b_dtype == 'float16':
            self.check_forward_options.update({
                'rtol': 1e-2, 'atol': 1e-2})
            self.check_backward_options.update({
                'rtol': 1e-2, 'atol': 1e-2})
            self.check_double_backward_options.update({
                'rtol': 1e-2, 'atol': 1e-2})

    def generate_inputs(self):
        a_dtype, b_dtype = self.in_dtypes
        a_shape = self.a_shape
        b_shape = self.b_shape
        a = numpy.random.uniform(-1, 1, a_shape).astype(a_dtype)
        b = numpy.random.uniform(-1, 1, b_shape).astype(b_dtype)
        return a, b

    def forward_xp(self, inputs, xp):
        a, b = inputs
        if self.is_module:
            y = xp.dot(a, b)
        else:
            y = a.dot(b)
        y = dtype_utils.cast_if_numpy_array(xp, y, self.chx_expected_dtype)
        return y,

@chainerx.testing.numpy_chainerx_array_equal(
    accept_error=(chainerx.DimensionError, ValueError))
@pytest.mark.parametrize('a_shape,b_shape', [
    ((3, 2), (1, 3)),
    ((4, 3, 2, 5), (6, 4, 1, 2))
])
@pytest.mark.parametrize_device(['native:0', 'cuda:0'])
def test_dot_invalid(is_module, xp, device, a_shape, b_shape, dtype):
    # TODO(beam2d): Remove the skip after supporting non-float dot on CUDA
    if device.name == 'cuda:0' and numpy.dtype(dtype).kind != 'f':
        return chainerx.testing.ignore()
    a = array_utils.create_dummy_ndarray(xp, a_shape, dtype)
    b = array_utils.create_dummy_ndarray(xp, b_shape, dtype)
    if is_module:
        return xp.dot(a, b)
    else:
        return a.dot(b)
<commit_msg>Added SVD test<commit_after>import numpy
import pytest

import chainer
import chainerx
import chainerx.testing

from chainerx_tests import array_utils
from chainerx_tests import dtype_utils
from chainerx_tests import op_utils

@op_utils.op_test(['native:0', 'cuda:0'])
@chainer.testing.parameterize_pytest('a_shape,b_shape', [
    ((), ()),
    ((), (2, 3)),
    ((2, 0), (0, 3)),
    ((0, 0), (0, 0)),
    ((2, 3), (3, 4)),
    ((4, 5, 2), (3, 2, 5)),
    ((2, 3, 4, 4), (3, 4, 2)),
    ((2, 2, 3, 1), (2, 1, 3, 1,  4)),
    ((2, 4, 3), (1, 2, 3, 2))

    # TODO(niboshi): Add test cases for more than 2 ndim
])
@chainer.testing.parameterize_pytest(
    'in_dtypes,chx_expected_dtype', dtype_utils.result_dtypes_two_arrays)
@chainer.testing.parameterize_pytest('is_module', [True, False])
class TestDot(op_utils.NumpyOpTest):

    def setup(self):
        device = chainerx.get_default_device()
        a_dtype, b_dtype = self.in_dtypes
        a_kind = numpy.dtype(a_dtype).kind
        b_kind = numpy.dtype(b_dtype).kind
        # TODO(beam2d): Remove the skip after supporting non-float dot on CUDA
        if device.name == 'cuda:0' and (a_kind != 'f' and b_kind != 'f'):
            pytest.skip('non-float dot is not supported on CUDA')

        # Skip backward/double-backward tests for int dtypes
        if a_kind != 'f' or b_kind != 'f':
            self.skip_backward_test = True
            self.skip_double_backward_test = True
        # Skip backward/double-backward tests if the output will be
        # disconnected.
        # TODO(niboshi): Remove this skip condition after enabling backward()
        # for such cases.
        if self.a_shape and self.a_shape[-1] == 0:
            self.skip_backward_test = True
            self.skip_double_backward_test = True

        if a_dtype == 'float16' or b_dtype == 'float16':
            self.check_forward_options.update({
                'rtol': 1e-2, 'atol': 1e-2})
            self.check_backward_options.update({
                'rtol': 1e-2, 'atol': 1e-2})
            self.check_double_backward_options.update({
                'rtol': 1e-2, 'atol': 1e-2})

    def generate_inputs(self):
        a_dtype, b_dtype = self.in_dtypes
        a_shape = self.a_shape
        b_shape = self.b_shape
        a = numpy.random.uniform(-1, 1, a_shape).astype(a_dtype)
        b = numpy.random.uniform(-1, 1, b_shape).astype(b_dtype)
        return a, b

    def forward_xp(self, inputs, xp):
        a, b = inputs
        if self.is_module:
            y = xp.dot(a, b)
        else:
            y = a.dot(b)
        y = dtype_utils.cast_if_numpy_array(xp, y, self.chx_expected_dtype)
        return y,

@chainerx.testing.numpy_chainerx_array_equal(
    accept_error=(chainerx.DimensionError, ValueError))
@pytest.mark.parametrize('a_shape,b_shape', [
    ((3, 2), (1, 3)),
    ((4, 3, 2, 5), (6, 4, 1, 2))
])
@pytest.mark.parametrize_device(['native:0', 'cuda:0'])
def test_dot_invalid(is_module, xp, device, a_shape, b_shape, dtype):
    # TODO(beam2d): Remove the skip after supporting non-float dot on CUDA
    if device.name == 'cuda:0' and numpy.dtype(dtype).kind != 'f':
        return chainerx.testing.ignore()
    a = array_utils.create_dummy_ndarray(xp, a_shape, dtype)
    b = array_utils.create_dummy_ndarray(xp, b_shape, dtype)
    if is_module:
        return xp.dot(a, b)
    else:
        return a.dot(b)

@op_utils.op_test(['native:0', 'cuda:0'])
@chainer.testing.parameterize(*(
    chainer.testing.product({
        'shape': [(1, 1), (2, 3), (5, 5)],
        'in_dtypes': ['float32', 'float64'],
        'full_matrices': [True, False],
        'compute_uv': [True, False],
        'skip_backward_test': [True],
        'skip_double_backward_test': [True],
    })
))
class TestSVD(op_utils.NumpyOpTest):

    def setup(self):
        device = chainerx.get_default_device()
        if device.name == 'native:0':
            pytest.skip('CPU SVD is not implemented')

    def generate_inputs(self):
        a = numpy.random.random(self.shape).astype(self.in_dtypes)
        return a,

    def forward_xp(self, inputs, xp):
        a, = inputs
        out = xp.linalg.svd(a, full_matrices=self.full_matrices, compute_uv=self.compute_uv)
        # NOTE: cuSOLVER's (CuPy's) and NumPy's outputs of u and v might
        # differ in signs, which is not a problem mathematically
        if self.compute_uv:
            u, s, v = out
            return xp.abs(u), s, xp.abs(v)
        else:
            if xp is chainerx:
                _, s, _ = out
            else:
                s = out
            return s,
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
import sys
import unittest

from mako import exceptions, util
from mako.template import Template
from mako.lookup import TemplateLookup
from util import result_lines
from test import template_base, module_base, TemplateTest

class ExceptionsTest(TemplateTest):
    def test_html_error_template(self):
        """test the html_error_template"""
        code = """
% i = 0
"""
        try:
            template = Template(code)
            template.render_unicode()
            assert False
        except exceptions.CompileException, ce:
            html_error = exceptions.html_error_template().render_unicode()
            assert ("CompileException: Fragment 'i = 0' is not "
                    "a partial control statement at line: 2 char: 1") in html_error
            assert '


<commit_before>from crispy_forms.helper import FormHelper
from dal import autocomplete
from django import forms
from django.contrib.auth.models import AnonymousUser
from django.forms import DateField, DateInput
from django.urls import reverse
from django.utils.html import format_html

from core.layout import *
from projects.models import Project, Nomination, Claim
from core.models import Resource
from core.widgets import ResourceInput

class ProjectForm(forms.ModelForm):
    """Project model form."""

    class Meta:
        """Metaclass for options."""

        model = Project
        fields = ['title', 'description', 'status', 'administrators',
                  'nomination_policy', 'nominators', 'nominator_blacklist',
                  'tags']
        widgets = {
            'title': forms.TextInput,
            'administrators': autocomplete.ModelSelect2Multiple(
                url='user_autocomplete',
                attrs={'data-allow-clear': 'false',
                       'data-width': '100%'},
            ),
            'nominators': autocomplete.ModelSelect2Multiple(
                url='user_autocomplete',
                attrs={'data-allow-clear': 'false',
                       'data-width': '100%'},
            ),
            'nominator_blacklist': autocomplete.ModelSelect2Multiple(
                url='user_autocomplete',
                attrs={'data-allow-clear': 'false',
                       'data-width': '100%'},
            ),
            'tags': autocomplete.ModelSelect2Multiple(
                url='tag_autocomplete',
                attrs={'data-allow-clear': 'false',
                       'data-width': '100%'},
            ),
        }

    def __init__(self, *args, editable=False,  **kwargs):
        """Initialize ProjectForm, adding crispy_forms helper and layout."""

        super().__init__(*args, **kwargs)
        self.helper = FormHelper(self)

        if hasattr(self.instance, 'pk') and self.instance.pk is not None:
            form_buttons_kwargs = {
                'confirm_title': 'Save changes',
                'confirm_text': 'Click the submit button to save changes to this project or click on cancel to return to Cobweb without saving.',
            }
        else:
            form_buttons_kwargs = {
                'confirm_title': 'Add new project',
                'confirm_text': 'Click the submit button to add this project to Cobweb or click on cancel to return to Cobweb without saving.',
            }

        self.helper.layout = Layout(
            HField('title', edit=editable),

            FormSection(
                Row(Column(HField('description', edit=editable))),

                Row(
                    Field('status', edit=editable, wrapper_class='col-md-5'),
                    Field('administrators', edit=editable, wrapper_class='col-md-7'),
                ),
            ),

            FormSection(
                Row(
                    Column(Field('nomination_policy', edit=editable), css_class='col-md-5'),
                    Column(
                        Field('nominators', edit=editable),
                        Field('nominator_blacklist', edit=editable, show=editable),
                        css_class='col-md-7'
                    ),
                ),
            ),

            FormSection(Row(Column(Field('tags', edit=editable)))),

            form_buttons(**form_buttons_kwargs) if editable else HTML(''),
        )

CRAWL_SCOPE_FIELD_NAMES = [
    'crawl_start_date',
    'crawl_end_date',
    'crawl_frequency',
    'follow_links',
    'page_scope',
    'ignore_robots_txt',
    'rights_considerations',
]

def nomination_info(editable=False):
    return Layout(
        FormSection(
            Field('project', type='hidden'),
            Field('nominated_by', type='hidden'),
            Row(Column(Field('rationale', edit=editable))),
        ),
        FormSection(crawl_scope_fields(editable=editable)),
    )

def resource_info(editable=False, instance: Optional[Resource]=None):
    edit_form = Layout(
        HField('resource', edit=editable),
        HField('title', edit=editable),
        HField('description', edit=editable),
        Field('tags', edit=editable),
        Field('language', edit=editable),
    )

    if (instance and hasattr(instance, 'resource') and instance.resource.has_metadata):
        return info_tabs(
            InfoTab(title='All metadata',
                    content=HTML("""
                        {% load react %}
                        {% react_render component="components.Resource" props=react_data %}

                        {% block load_react_script %}
                        {% react_print %}
                        {% endblock %}
                    """)),
            InfoTab(title="This project's submission",
                    content=edit_form),
        )
    else:
        return edit_form

class NominationForm(forms.ModelForm):
    class Meta:
        model = Nomination
        fields = (['resource', 'title', 'description', 'tags', 'language',
                   'project', 'nominated_by', 'rationale']
                  + CRAWL_SCOPE_FIELD_NAMES)
        widgets = {
            'tags': autocomplete.ModelSelect2Multiple(
                url='tag_autocomplete',
                attrs={'data-allow-clear': 'false'},
            ),
            'crawl_start_date': DateInput,
            'crawl_end_date': DateInput,
        }

    resource = forms.URLField(widget=ResourceInput, initial='http://')

    def __init__(self, *args, table=None, editable=False, tabbed=False, instance=None,
                 **kwargs):
        super().__init__(*args, instance=instance, **kwargs)
        self.helper = FormHelper(self)

        if hasattr(self.instance, 'pk') and self.instance.pk is not None:
            form_title = HTML('
{% load jargon %}{% term "nomination" "upper" %}: {{object.resource}}
')
            form_buttons_kwargs = {
                'confirm_title': 'Save changes',
                'confirm_text': 'Click the submit button to save changes to this nomination or click on cancel to return to Cobweb without saving.',
            }
        else:
            form_title = HTML('
New {% load jargon %}{% term "nomination" "lower" %}
')
            form_buttons_kwargs = {
                'confirm_title': 'Add new nomination',
                'confirm_text': 'Click the submit button to nominate this resource or click on cancel to return to Cobweb without saving.',
            }

        if 'project' in self.initial:
            # if not (hasattr(self, 'project') and isinstance(self.project, Project)):
            #     self.project = Project.objects.get(pk=self.initial['project'])
            proj_header = HTML("""
                {% load jargon cobweb_look %}

{% term "project" "capitalize" %}: {{view.get_project|as_link}}

            """)
        else:
            proj_header = HTML('')

        if tabbed:
            self.helper.layout = info_tabs(
                InfoTab(title='About the nomination',
                        content=nomination_info(editable=editable)),
                InfoTab(title='About the resource',
                        content=resource_info(editable=editable, instance=self.instance)),
            )
        else:
            self.helper.layout = Layout(
                Div(
                    form_title,
                    proj_header,
                    css_class='px-3 pt-0 pb-2 w-100',
                ),

                Row(
                    Pane(
                        Row(Column(HTML('
About the nomination
'))),
                        nomination_info(editable=editable),
                        HTML("""
                            {% load render_table from django_tables2 %}

Claims

                            {% render_table table %}
                        """ if table else ''),
                        css_class='col-6',
                    ),
                    Pane(
                        Row(Column(HTML('
About the resource
'))),
                        resource_info(editable=editable, instance=self.instance),
                        form_buttons(**form_buttons_kwargs),
                        css_class='col-6',
                    ),
                    css_class='flex-grow-1',
                )
            )
        self.helper.form_class = 'h-100 d-flex flex-column pb-2'

    def clean_resource(self):
        url = self.cleaned_data.get("resource")
        if not url:
            raise forms.ValidationError("Please enter a URL.")
        else:
            return Resource.objects.get_or_create(url=url)[0]

class ClaimForm(forms.ModelForm):
    class Meta:
        model = Claim
        fields = (['nomination', 'organization', 'active', 'has_holding']
                  + CRAWL_SCOPE_FIELD_NAMES)
        widgets = {
            'tags': autocomplete.ModelSelect2Multiple(
                url='tag_autocomplete',
                attrs={'data-allow-clear': 'false'},
            ),
            'crawl_start_date': DateInput,
            'crawl_end_date': DateInput,
        }

    def __init__(self, *args, editable=False, **kwargs) -> None:
        super().__init__(*args, **kwargs)

        self.fields['organization'].queryset = self.fields['organization'].queryset.filter(
            pk=self.instance.organization.pk
        )

        self.helper = FormHelper(self)
        self.helper.layout = Layout(
            FormSection(Row(Column(
                HTML('
About the {% load jargon %}{% term "claim" %}
')
            ))),
            FormSection(Row(Column(
                HTML("""{% load as_link from cobweb_look %}
                    Nomination:

Resource URL: {{form.instance.nomination.resource|as_link}}

Project: {{form.instance.nomination.project|as_link}}
                """),
                Hidden('nomination', value=self.initial['nomination']),
            )), css_class='form-group'),
            FormSection(
                Row(Column(HField('organization', edit=editable))),
                Row(Column(Field('active', edit=editable), css_class='col-6'),
                    Column(Field('has_holding', edit=editable), css_class='col-6')),
            ),
            crawl_scope_fields(editable=editable),
            form_buttons(
                confirm_title='Claim nominated URL',
                confirm_text='Click the claim button to claim the nominated URL to the project or cancel to return to Cobweb',
            ) if editable else HTML(''),
        )
<commit_msg>ui: rearranged nomination view<commit_after>from crispy_forms.helper import FormHelper
from dal import autocomplete
from django import forms
from django.contrib.auth.models import AnonymousUser
from django.forms import DateField, DateInput
from django.urls import reverse
from django.utils.html import format_html

from core.layout import *
from projects.models import Project, Nomination, Claim
from core.models import Resource
from core.widgets import ResourceInput

class ProjectForm(forms.ModelForm):
    """Project model form."""

    class Meta:
        """Metaclass for options."""

        model = Project
        fields = ['title', 'description', 'status', 'administrators',
                  'nomination_policy', 'nominators', 'nominator_blacklist',
                  'tags']
        widgets = {
            'title': forms.TextInput,
            'administrators': autocomplete.ModelSelect2Multiple(
                url='user_autocomplete',
                attrs={'data-allow-clear': 'false',
                       'data-width': '100%'},
            ),
            'nominators': autocomplete.ModelSelect2Multiple(
                url='user_autocomplete',
                attrs={'data-allow-clear': 'false',
                       'data-width': '100%'},
            ),
            'nominator_blacklist': autocomplete.ModelSelect2Multiple(
                url='user_autocomplete',
                attrs={'data-allow-clear': 'false',
                       'data-width': '100%'},
            ),
            'tags': autocomplete.ModelSelect2Multiple(
                url='tag_autocomplete',
                attrs={'data-allow-clear': 'false',
                       'data-width': '100%'},
            ),
        }

    def __init__(self, *args, editable=False,  **kwargs):
        """Initialize ProjectForm, adding crispy_forms helper and layout."""

        super().__init__(*args, **kwargs)
        self.helper = FormHelper(self)

        if hasattr(self.instance, 'pk') and self.instance.pk is not None:
            form_buttons_kwargs = {
                'confirm_title': 'Save changes',
                'confirm_text': 'Click the submit button to save changes to this project or click on cancel to return to Cobweb without saving.',
            }
        else:
            form_buttons_kwargs = {
                'confirm_title': 'Add new project',
                'confirm_text': 'Click the submit button to add this project to Cobweb or click on cancel to return to Cobweb without saving.',
            }

        self.helper.layout = Layout(
            HField('title', edit=editable),

            FormSection(
                Row(Column(HField('description', edit=editable))),

                Row(
                    Field('status', edit=editable, wrapper_class='col-md-5'),
                    Field('administrators', edit=editable, wrapper_class='col-md-7'),
                ),
            ),

            FormSection(
                Row(
                    Column(Field('nomination_policy', edit=editable), css_class='col-md-5'),
                    Column(
                        Field('nominators', edit=editable),
                        Field('nominator_blacklist', edit=editable, show=editable),
                        css_class='col-md-7'
                    ),
                ),
            ),

            FormSection(Row(Column(Field('tags', edit=editable)))),

            form_buttons(**form_buttons_kwargs) if editable else HTML(''),
        )

CRAWL_SCOPE_FIELD_NAMES = [
    'crawl_start_date',
    'crawl_end_date',
    'crawl_frequency',
    'follow_links',
    'page_scope',
    'ignore_robots_txt',
    'rights_considerations',
]

def nomination_info(editable=False):
    return Layout(
        FormSection(
            Field('project', type='hidden'),
            Field('nominated_by', type='hidden'),
            Row(Column(Field('rationale', edit=editable))),
        ),
        FormSection(crawl_scope_fields(editable=editable)),
    )

def resource_info(editable=False, instance: Optional[Resource]=None):
    edit_form = Layout(
        HField('resource', edit=editable),
        HField('title', edit=editable),
        HField('description', edit=editable),
        Field('tags', edit=editable),
        Field('language', edit=editable),
    )

    if (instance and hasattr(instance, 'resource') and instance.resource.has_metadata):
        return info_tabs(
            InfoTab(title='All metadata',
                    content=HTML("""
                        {% load react %}
                        {% react_render component="components.Resource" props=react_data %}

                        {% block load_react_script %}
                        {% react_print %}
                        {% endblock %}
                    """)),
            InfoTab(title="This project's submission",
                    content=edit_form),
        )
    else:
        return edit_form

class NominationForm(forms.ModelForm):
    class Meta:
        model = Nomination
        fields = (['resource', 'title', 'description', 'tags', 'language',
                   'project', 'nominated_by', 'rationale']
                  + CRAWL_SCOPE_FIELD_NAMES)
        widgets = {
            'tags': autocomplete.ModelSelect2Multiple(
                url='tag_autocomplete',
                attrs={'data-allow-clear': 'false'},
            ),
            'crawl_start_date': DateInput,
            'crawl_end_date': DateInput,
        }

    resource = forms.URLField(widget=ResourceInput, initial='http://')

    def __init__(self, *args, table=None, editable=False, tabbed=False, instance=None,
                 **kwargs):
        super().__init__(*args, instance=instance, **kwargs)
        self.helper = FormHelper(self)

        if hasattr(self.instance, 'pk') and self.instance.pk is not None:
            form_title = HTML('
{% load jargon %}{% term "nomination" "upper" %}: {{object.resource}}
')
            form_buttons_kwargs = {
                'confirm_title': 'Save changes',
                'confirm_text': 'Click the submit button to save changes to this nomination or click on cancel to return to Cobweb without saving.',
            }
        else:
            form_title = HTML('
New {% load jargon %}{% term "nomination" "lower" %}
')
            form_buttons_kwargs = {
                'confirm_title': 'Add new nomination',
                'confirm_text': 'Click the submit button to nominate this resource or click on cancel to return to Cobweb without saving.',
            }

        if 'project' in self.initial:
            # if not (hasattr(self, 'project') and isinstance(self.project, Project)):
            #     self.project = Project.objects.get(pk=self.initial['project'])
            proj_header = HTML("""
                {% load jargon cobweb_look %}

{% term "project" "capitalize" %}: {{view.get_project|as_link}}

            """)
        else:
            proj_header = HTML('')

        if tabbed:
            self.helper.layout = info_tabs(
                InfoTab(title='About the nomination',
                        content=nomination_info(editable=editable)),
                InfoTab(title='About the resource',
                        content=resource_info(editable=editable, instance=self.instance)),
            )
        else:
            self.helper.layout = Layout(
                Div(
                    form_title,
                    proj_header,
                    css_class='px-3 pt-0 pb-2 w-100',
                ),

                Row(
                    Pane(
                        Row(Column(HTML('
About the resource
'))),
                        resource_info(editable=editable, instance=self.instance),
                        css_class='col-6',
                    ),
                    Pane(
                        Row(Column(HTML('
About the nomination
'))),
                        nomination_info(editable=editable),
                        form_buttons(**form_buttons_kwargs),
                        HTML("""
                            {% load render_table from django_tables2 %}

Claims

                            {% render_table table %}
                        """ if table else ''),
                        css_class='col-6',
                    ),
                    css_class='flex-grow-1',
                )
            )
        self.helper.form_class = 'h-100 d-flex flex-column pb-2'

    def clean_resource(self):
        url = self.cleaned_data.get("resource")
        if not url:
            raise forms.ValidationError("Please enter a URL.")
        else:
            return Resource.objects.get_or_create(url=url)[0]

class ClaimForm(forms.ModelForm):
    class Meta:
        model = Claim
        fields = (['nomination', 'organization', 'active', 'has_holding']
                  + CRAWL_SCOPE_FIELD_NAMES)
        widgets = {
            'tags': autocomplete.ModelSelect2Multiple(
                url='tag_autocomplete',
                attrs={'data-allow-clear': 'false'},
            ),
            'crawl_start_date': DateInput,
            'crawl_end_date': DateInput,
        }

    def __init__(self, *args, editable=False, **kwargs) -> None:
        super().__init__(*args, **kwargs)

        self.fields['organization'].queryset = self.fields['organization'].queryset.filter(
            pk=self.instance.organization.pk
        )

        self.helper = FormHelper(self)
        self.helper.layout = Layout(
            FormSection(Row(Column(
                HTML('
About the {% load jargon %}{% term "claim" %}
')
            ))),
            FormSection(Row(Column(
                HTML("""{% load as_link from cobweb_look %}
                    Nomination:

Resource URL: {{form.instance.nomination.resource|as_link}}

Project: {{form.instance.nomination.project|as_link}}
                """),
                Hidden('nomination', value=self.initial['nomination']),
            )), css_class='form-group'),
            FormSection(
                Row(Column(HField('organization', edit=editable))),
                Row(Column(Field('active', edit=editable), css_class='col-6'),
                    Column(Field('has_holding', edit=editable), css_class='col-6')),
            ),
            crawl_scope_fields(editable=editable),
            form_buttons(
                confirm_title='Claim nominated URL',
                confirm_text='Click the claim button to claim the nominated URL to the project or cancel to return to Cobweb',
            ) if editable else HTML(''),
        )
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from __future__ import unicode_literals

import logging
import shutil
import os

from django.conf import settings
from django.contrib.auth.models import User
from django.db import models
from django.db.models.signals import pre_delete, post_delete
from django.dispatch.dispatcher import receiver
from django.utils import timezone

from ..jobs.models import Job, LowerCaseCharField
from ..utils.s3 import delete_from_s3
from ..core.models import UIDMixin, TimeStampedModelMixin, TimeTrackingModelMixin
import uuid

logger = logging.getLogger(__name__)

class ExportRun(UIDMixin, TimeStampedModelMixin, TimeTrackingModelMixin):
    """
    ExportRun is the main structure for storing export information.

    A Job provides information for the ExportRun.
    Many ExportRuns can map to a Job.
    Many ExportProviderTasks can map to an ExportRun.
    Many ExportTasks can map to an DataProviderTaskRecord.
    """
    job = models.ForeignKey(Job, related_name='runs')
    user = models.ForeignKey(User, related_name="runs", default=0)
    worker = models.CharField(max_length=50, editable=False, default='', null=True)
    zipfile_url = models.CharField(max_length=1000, db_index=False, blank=True, null=True)
    status = models.CharField(
        blank=True,
        max_length=20,
        db_index=True,
        default=''
    )
    expiration = models.DateTimeField(default=timezone.now, editable=True)
    notified = models.DateTimeField(default=None, blank=True, null=True)
    deleted = models.BooleanField(default=False)
    delete_user = models.ForeignKey(User, null=True, blank=True, editable=False)

    class Meta:
        managed = True
        db_table = 'export_runs'

    def __str__(self):
        return '{0}'.format(self.uid)

    def soft_delete(self, user=None, *args, **kwargs):
        from .export_tasks import cancel_run
        exportrun_delete_exports(self.__class__, self)
        username = None
        if user:
            self.delete_user = user
            username = user.username
        self.deleted = True
        logger.info("Deleting run {0} by user {1}".format(self.uid, user))
        cancel_run(export_run_uid=self.uid, canceling_username=username, delete=True)
        self.save()

class DataProviderTaskRecord(UIDMixin, TimeStampedModelMixin, TimeTrackingModelMixin):
    """
    The DataProviderTaskRecord stores the task information for a specific provider.
    """
    name = models.CharField(max_length=50, blank=True)
    slug = LowerCaseCharField(max_length=40, default='')
    run = models.ForeignKey(ExportRun, related_name='provider_tasks')
    status = models.CharField(blank=True, max_length=20, db_index=True)
    display = models.BooleanField(default=False)

    class Meta:
        ordering = ['name']
        managed = True
        db_table = 'data_provider_task_records'

    def __str__(self):
        return 'DataProviderTaskRecord uid: {0}'.format(self.uid)

class ExportTaskRecord(UIDMixin, TimeStampedModelMixin, TimeTrackingModelMixin):
    """
     An ExportTaskRecord holds the information about the process doing the actual work for a task.
    """
    celery_uid = models.UUIDField(null=True)  # celery task uid
    name = models.CharField(max_length=50)
    export_provider_task = models.ForeignKey(DataProviderTaskRecord, related_name='tasks')
    status = models.CharField(blank=True, max_length=20, db_index=True)
    progress = models.IntegerField(default=0, editable=False, null=True)
    estimated_finish = models.DateTimeField(blank=True, editable=False, null=True)
    pid = models.IntegerField(blank=True, default=-1)
    worker = models.CharField(max_length=100, blank=True, editable=False, null=True)
    cancel_user = models.ForeignKey(User, null=True, blank=True, editable=False)
    display = models.BooleanField(default=False)
    result = models.OneToOneField('FileProducingTaskResult', null=True, blank=True, related_name='export_task')

    class Meta:
        ordering = ['created_at']
        managed = True
        db_table = 'export_task_records'

    def __str__(self):
        return 'ExportTaskRecord uid: {0}'.format(self.uid)

class FinalizeRunHookTaskRecord(UIDMixin, TimeStampedModelMixin):
    run = models.ForeignKey(ExportRun)
    celery_uid = models.UUIDField()
    task_name = models.CharField(max_length=50)
    status = models.CharField(blank=True, max_length=20, db_index=True)
    pid = models.IntegerField(blank=True, default=-1)
    worker = models.CharField(max_length=100, blank=True, editable=False, null=True)
    cancel_user = models.ForeignKey(User, null=True, blank=True, editable=False)
    result = models.OneToOneField('FileProducingTaskResult', null=True, blank=True, related_name='finalize_task')

    class Meta:
        ordering = ['created_at']
        managed = True
        db_table = 'finalize_run_hook_task_record'

    def __str__(self):
        return 'RunFinishedTaskRecord ({}): {}'.format(self.celery_uid, self.status)

class FileProducingTaskResult(models.Model):
    """
         A FileProducingTaskResult holds the information from the task, i.e. the reason for executing the task.
    """
    id = models.AutoField(primary_key=True)
    filename = models.CharField(max_length=100, blank=True, editable=False)
    size = models.FloatField(null=True, editable=False)
    download_url = models.URLField(
        verbose_name='URL to export task result output.',
        max_length=254
    )
    deleted = models.BooleanField(default=False)

    @property
    def task(self):
        if hasattr(self, 'finalize_task') and hasattr(self.export_task):
            raise Exception('Both an ExportTaskRecord and a FinalizeRunHookTaskRecord are linked to FileProducingTaskResult')
        elif hasattr(self, 'finalize_task'):
            ret = self.finalize_task
        elif hasattr(self, 'export_task'):
            ret = self.export_task
        else:
            ret = None
        return ret

    def soft_delete(self, *args, **kwargs):
        exporttaskresult_delete_exports(self.__class__, self)
        self.deleted = True
        self.save()
        if hasattr(self.task, 'display'):
            self.task.display = False
            self.task.save()

    class Meta:
        managed = True
        db_table = 'export_task_results'

    def __str__(self):
        return 'FileProducingTaskResult ({}), {}'.format(self.id, self.filename)

class ExportTaskException(TimeStampedModelMixin):
    """
    Model to store ExportTaskRecord exceptions for auditing.
    """
    id = models.AutoField(primary_key=True, editable=False)
    task = models.ForeignKey(ExportTaskRecord, related_name='exceptions')
    exception = models.TextField(editable=False)

    class Meta:
        managed = True
        db_table = 'export_task_exceptions'

@receiver(pre_delete, sender=ExportRun)
def exportrun_delete_exports(sender, instance, *args, **kwargs):
    """
    Delete the associated export files when an ExportRun is deleted.
    """
    if getattr(settings, 'USE_S3', False):
        delete_from_s3(run_uid=str(instance.uid))
    run_dir = '{0}/{1}'.format(settings.EXPORT_DOWNLOAD_ROOT.rstrip('/'), instance.uid)
    try:
        shutil.rmtree(run_dir, ignore_errors=True)
        logger.info("The directory {0} was deleted.".format(run_dir))
    except OSError:
        logger.warn("The directory {0} was already moved or doesn't exist.".format(run_dir))

@receiver(pre_delete, sender=FileProducingTaskResult)
def exporttaskresult_delete_exports(sender, instance, *args, **kwargs):
    """
    Delete associated files when deleting the FileProducingTaskResult.
    """
    # The url should be constructed as [download context, run_uid, filename]
    if getattr(settings, 'USE_S3', False):
        delete_from_s3(download_url=instance.download_url)
    url_parts = instance.download_url.split('/')
    full_file_download_path = '/'.join([settings.EXPORT_DOWNLOAD_ROOT.rstrip('/'), url_parts[-2], url_parts[-1]])
    try:
        os.remove(full_file_download_path)
        logger.info("The directory {0} was deleted.".format(full_file_download_path))
    except OSError:
        logger.warn("The file {0} was already removed or does not exist.".format(full_file_download_path))
<commit_msg>Verbose name<commit_after># -*- coding: utf-8 -*-
from __future__ import unicode_literals

import logging
import shutil
import os

from django.conf import settings
from django.contrib.auth.models import User
from django.db import models
from django.db.models.signals import pre_delete, post_delete
from django.dispatch.dispatcher import receiver
from django.utils import timezone

from ..jobs.models import Job, LowerCaseCharField
from ..utils.s3 import delete_from_s3
from ..core.models import UIDMixin, TimeStampedModelMixin, TimeTrackingModelMixin
import uuid

logger = logging.getLogger(__name__)

class ExportRun(UIDMixin, TimeStampedModelMixin, TimeTrackingModelMixin):
    """
    ExportRun is the main structure for storing export information.

    A Job provides information for the ExportRun.
    Many ExportRuns can map to a Job.
    Many ExportProviderTasks can map to an ExportRun.
    Many ExportTasks can map to an DataProviderTaskRecord.
    """
    job = models.ForeignKey(Job, related_name='runs')
    user = models.ForeignKey(User, related_name="runs", default=0)
    worker = models.CharField(max_length=50, editable=False, default='', null=True)
    zipfile_url = models.CharField(max_length=1000, db_index=False, blank=True, null=True)
    status = models.CharField(
        blank=True,
        max_length=20,
        db_index=True,
        default=''
    )
    expiration = models.DateTimeField(default=timezone.now, editable=True)
    notified = models.DateTimeField(default=None, blank=True, null=True)
    deleted = models.BooleanField(default=False)
    delete_user = models.ForeignKey(User, null=True, blank=True, editable=False)

    class Meta:
        managed = True
        db_table = 'export_runs'
        verbose_name = 'ExportRun (DataPack)'
        verbose_name_plural = 'ExportRuns (DataPacks)'

    def __str__(self):
        return '{0}'.format(self.uid)

    def soft_delete(self, user=None, *args, **kwargs):
        from .export_tasks import cancel_run
        exportrun_delete_exports(self.__class__, self)
        username = None
        if user:
            self.delete_user = user
            username = user.username
        self.deleted = True
        logger.info("Deleting run {0} by user {1}".format(self.uid, user))
        cancel_run(export_run_uid=self.uid, canceling_username=username, delete=True)
        self.save()

class DataProviderTaskRecord(UIDMixin, TimeStampedModelMixin, TimeTrackingModelMixin):
    """
    The DataProviderTaskRecord stores the task information for a specific provider.
    """
    name = models.CharField(max_length=50, blank=True)
    slug = LowerCaseCharField(max_length=40, default='')
    run = models.ForeignKey(ExportRun, related_name='provider_tasks')
    status = models.CharField(blank=True, max_length=20, db_index=True)
    display = models.BooleanField(default=False)

    class Meta:
        ordering = ['name']
        managed = True
        db_table = 'data_provider_task_records'

    def __str__(self):
        return 'DataProviderTaskRecord uid: {0}'.format(self.uid)

class ExportTaskRecord(UIDMixin, TimeStampedModelMixin, TimeTrackingModelMixin):
    """
     An ExportTaskRecord holds the information about the process doing the actual work for a task.
    """
    celery_uid = models.UUIDField(null=True)  # celery task uid
    name = models.CharField(max_length=50)
    export_provider_task = models.ForeignKey(DataProviderTaskRecord, related_name='tasks')
    status = models.CharField(blank=True, max_length=20, db_index=True)
    progress = models.IntegerField(default=0, editable=False, null=True)
    estimated_finish = models.DateTimeField(blank=True, editable=False, null=True)
    pid = models.IntegerField(blank=True, default=-1)
    worker = models.CharField(max_length=100, blank=True, editable=False, null=True)
    cancel_user = models.ForeignKey(User, null=True, blank=True, editable=False)
    display = models.BooleanField(default=False)
    result = models.OneToOneField('FileProducingTaskResult', null=True, blank=True, related_name='export_task')

    class Meta:
        ordering = ['created_at']
        managed = True
        db_table = 'export_task_records'

    def __str__(self):
        return 'ExportTaskRecord uid: {0}'.format(self.uid)

class FinalizeRunHookTaskRecord(UIDMixin, TimeStampedModelMixin):
    run = models.ForeignKey(ExportRun)
    celery_uid = models.UUIDField()
    task_name = models.CharField(max_length=50)
    status = models.CharField(blank=True, max_length=20, db_index=True)
    pid = models.IntegerField(blank=True, default=-1)
    worker = models.CharField(max_length=100, blank=True, editable=False, null=True)
    cancel_user = models.ForeignKey(User, null=True, blank=True, editable=False)
    result = models.OneToOneField('FileProducingTaskResult', null=True, blank=True, related_name='finalize_task')

    class Meta:
        ordering = ['created_at']
        managed = True
        db_table = 'finalize_run_hook_task_record'

    def __str__(self):
        return 'RunFinishedTaskRecord ({}): {}'.format(self.celery_uid, self.status)

class FileProducingTaskResult(models.Model):
    """
         A FileProducingTaskResult holds the information from the task, i.e. the reason for executing the task.
    """
    id = models.AutoField(primary_key=True)
    filename = models.CharField(max_length=100, blank=True, editable=False)
    size = models.FloatField(null=True, editable=False)
    download_url = models.URLField(
        verbose_name='URL to export task result output.',
        max_length=254
    )
    deleted = models.BooleanField(default=False)

    @property
    def task(self):
        if hasattr(self, 'finalize_task') and hasattr(self.export_task):
            raise Exception('Both an ExportTaskRecord and a FinalizeRunHookTaskRecord are linked to FileProducingTaskResult')
        elif hasattr(self, 'finalize_task'):
            ret = self.finalize_task
        elif hasattr(self, 'export_task'):
            ret = self.export_task
        else:
            ret = None
        return ret

    def soft_delete(self, *args, **kwargs):
        exporttaskresult_delete_exports(self.__class__, self)
        self.deleted = True
        self.save()
        if hasattr(self.task, 'display'):
            self.task.display = False
            self.task.save()

    class Meta:
        managed = True
        db_table = 'export_task_results'

    def __str__(self):
        return 'FileProducingTaskResult ({}), {}'.format(self.id, self.filename)

class ExportTaskException(TimeStampedModelMixin):
    """
    Model to store ExportTaskRecord exceptions for auditing.
    """
    id = models.AutoField(primary_key=True, editable=False)
    task = models.ForeignKey(ExportTaskRecord, related_name='exceptions')
    exception = models.TextField(editable=False)

    class Meta:
        managed = True
        db_table = 'export_task_exceptions'

@receiver(pre_delete, sender=ExportRun)
def exportrun_delete_exports(sender, instance, *args, **kwargs):
    """
    Delete the associated export files when an ExportRun is deleted.
    """
    if getattr(settings, 'USE_S3', False):
        delete_from_s3(run_uid=str(instance.uid))
    run_dir = '{0}/{1}'.format(settings.EXPORT_DOWNLOAD_ROOT.rstrip('/'), instance.uid)
    try:
        shutil.rmtree(run_dir, ignore_errors=True)
        logger.info("The directory {0} was deleted.".format(run_dir))
    except OSError:
        logger.warn("The directory {0} was already moved or doesn't exist.".format(run_dir))

@receiver(pre_delete, sender=FileProducingTaskResult)
def exporttaskresult_delete_exports(sender, instance, *args, **kwargs):
    """
    Delete associated files when deleting the FileProducingTaskResult.
    """
    # The url should be constructed as [download context, run_uid, filename]
    if getattr(settings, 'USE_S3', False):
        delete_from_s3(download_url=instance.download_url)
    url_parts = instance.download_url.split('/')
    full_file_download_path = '/'.join([settings.EXPORT_DOWNLOAD_ROOT.rstrip('/'), url_parts[-2], url_parts[-1]])
    try:
        os.remove(full_file_download_path)
        logger.info("The directory {0} was deleted.".format(full_file_download_path))
    except OSError:
        logger.warn("The file {0} was already removed or does not exist.".format(full_file_download_path))
<|endoftext|>


<commit_before>#       Licensed to the Apache Software Foundation (ASF) under one
#       or more contributor license agreements.  See the NOTICE file
#       distributed with this work for additional information
#       regarding copyright ownership.  The ASF licenses this file
#       to you under the Apache License, Version 2.0 (the
#       "License"); you may not use this file except in compliance
#       with the License.  You may obtain a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#       Unless required by applicable law or agreed to in writing,
#       software distributed under the License is distributed on an
#       "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
#       KIND, either express or implied.  See the License for the
#       specific language governing permissions and limitations
#       under the License.

import logging
import HTMLParser

from pylons import tmpl_context as c, app_globals as g
from bson import ObjectId

from allura.lib import helpers as h
from allura.lib.decorators import task
from allura.lib import mail_util
from allura.lib import exceptions as exc

log = logging.getLogger(__name__)

smtp_client = mail_util.SMTPClient()

@task
def route_email(
    peer, mailfrom, rcpttos, data):
    '''Route messages according to their destination:

    <topic>@<mount_point>.<subproj2>.<subproj1>.<project>.projects.sourceforge.net
    gets sent to c.app.handle_message(topic, message)
    '''
    try:
        msg = mail_util.parse_message(data)
    except: # pragma no cover
        log.exception('Parse Error: (%r,%r,%r)', peer, mailfrom, rcpttos)
        return
    mail_user = mail_util.identify_sender(peer, mailfrom, msg['headers'], msg)
    with h.push_config(c, user=mail_user):
        log.info('Received email from %s', c.user.username)
        # For each of the addrs, determine the project/app and route appropriately
        for addr in rcpttos:
            try:
                userpart, project, app = mail_util.parse_address(addr)
                with h.push_config(c, project=project, app=app):
                    if not app.has_access(c.user, userpart):
                        log.info('Access denied for %s to mailbox %s', c.user, userpart)
                    else:
                        if msg['multipart']:
                            msg_hdrs = msg['headers']
                            for part in msg['parts']:
                                if part.get('content_type', '').startswith('multipart/'): continue
                                msg = dict(
                                    headers=dict(msg_hdrs, **part['headers']),
                                    message_id=part['message_id'],
                                    in_reply_to=part['in_reply_to'],
                                    references=part['references'],
                                    filename=part['filename'],
                                    content_type=part['content_type'],
                                    payload=part['payload'])
                                c.app.handle_message(userpart, msg)
                        else:
                            c.app.handle_message(userpart, msg)
            except exc.MailError, e:
                log.error('Error routing email to %s: %s', addr, e)
            except:
                log.exception('Error routing mail to %s', addr)

@task
def sendmail(fromaddr, destinations, text, reply_to, subject,
             message_id, in_reply_to=None):
    from allura import model as M
    addrs_plain = []
    addrs_html = []
    addrs_multi = []
    if fromaddr is None:
        fromaddr = u'noreply@in.sf.net'
    elif '@' not in fromaddr:
        log.warning('Looking up user with fromaddr: %s', fromaddr)
        user = M.User.query.get(_id=ObjectId(fromaddr), disabled=False)
        if not user:
            log.warning('Cannot find user with ID: %s', fromaddr)
            fromaddr = u'noreply@in.sf.net'
        else:
            fromaddr = user.email_address_header()
    # Divide addresses based on preferred email formats
    for addr in destinations:
        if mail_util.isvalid(addr):
            addrs_plain.append(addr)
        else:
            try:
                user = M.User.query.get(_id=ObjectId(addr), disabled=False)
                if not user:
                    log.warning('Cannot find user with ID: %s', addr)
                    continue
            except:
                log.exception('Error looking up user with ID: %r' % addr)
                continue
            addr = user.email_address_header()
            if not addr and user.email_addresses:
                addr = user.email_addresses[0]
                log.warning('User %s has not set primary email address, using %s',
                            user._id, addr)
            if not addr:
                log.error("User %s (%s) has not set any email address, can't deliver",
                          user._id, user.username)
                continue
            if user.get_pref('email_format') == 'plain':
                addrs_plain.append(addr)
            elif user.get_pref('email_format') == 'html':
                addrs_html.append(addr)
            else:
                addrs_multi.append(addr)
    htmlparser = HTMLParser.HTMLParser()
    plain_msg = mail_util.encode_email_part(htmlparser.unescape(text), 'plain')
    html_text = g.forge_markdown(email=True).convert(text)
    html_msg = mail_util.encode_email_part(html_text, 'html')
    multi_msg = mail_util.make_multipart_message(plain_msg, html_msg)
    smtp_client.sendmail(
        addrs_multi, fromaddr, reply_to, subject, message_id,
        in_reply_to, multi_msg)
    smtp_client.sendmail(
        addrs_plain, fromaddr, reply_to, subject, message_id,
        in_reply_to, plain_msg)
    smtp_client.sendmail(
        addrs_html, fromaddr, reply_to, subject, message_id,
        in_reply_to, html_msg)

@task
def sendsimplemail(
    fromaddr,
    toaddr,
    text,
    reply_to,
    subject,
    message_id,
    in_reply_to=None):
    from allura import model as M
    if fromaddr is None:
        fromaddr = u'noreply@in.sf.net'
    elif '@' not in fromaddr:
        log.warning('Looking up user with fromaddr: %s', fromaddr)
        user = M.User.query.get(_id=ObjectId(fromaddr), disabled=False)
        if not user:
            log.warning('Cannot find user with ID: %s', fromaddr)
            fromaddr = u'noreply@in.sf.net'
        else:
            fromaddr = user.email_address_header()
    plain_msg = mail_util.encode_email_part(text, 'plain')
    html_text = g.forge_markdown(email=True).convert(text)
    html_msg = mail_util.encode_email_part(html_text, 'html')
    multi_msg = mail_util.make_multipart_message(plain_msg, html_msg)
    smtp_client.sendmail(
        [toaddr], fromaddr, reply_to, subject, message_id,
        in_reply_to, multi_msg)
<commit_msg>[#5693]  ticket:382 unescape text email for sendsimplemail<commit_after>#       Licensed to the Apache Software Foundation (ASF) under one
#       or more contributor license agreements.  See the NOTICE file
#       distributed with this work for additional information
#       regarding copyright ownership.  The ASF licenses this file
#       to you under the Apache License, Version 2.0 (the
#       "License"); you may not use this file except in compliance
#       with the License.  You may obtain a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#       Unless required by applicable law or agreed to in writing,
#       software distributed under the License is distributed on an
#       "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
#       KIND, either express or implied.  See the License for the
#       specific language governing permissions and limitations
#       under the License.

import logging
import HTMLParser

from pylons import tmpl_context as c, app_globals as g
from bson import ObjectId

from allura.lib import helpers as h
from allura.lib.decorators import task
from allura.lib import mail_util
from allura.lib import exceptions as exc

log = logging.getLogger(__name__)

smtp_client = mail_util.SMTPClient()

@task
def route_email(
    peer, mailfrom, rcpttos, data):
    '''Route messages according to their destination:

    <topic>@<mount_point>.<subproj2>.<subproj1>.<project>.projects.sourceforge.net
    gets sent to c.app.handle_message(topic, message)
    '''
    try:
        msg = mail_util.parse_message(data)
    except: # pragma no cover
        log.exception('Parse Error: (%r,%r,%r)', peer, mailfrom, rcpttos)
        return
    mail_user = mail_util.identify_sender(peer, mailfrom, msg['headers'], msg)
    with h.push_config(c, user=mail_user):
        log.info('Received email from %s', c.user.username)
        # For each of the addrs, determine the project/app and route appropriately
        for addr in rcpttos:
            try:
                userpart, project, app = mail_util.parse_address(addr)
                with h.push_config(c, project=project, app=app):
                    if not app.has_access(c.user, userpart):
                        log.info('Access denied for %s to mailbox %s', c.user, userpart)
                    else:
                        if msg['multipart']:
                            msg_hdrs = msg['headers']
                            for part in msg['parts']:
                                if part.get('content_type', '').startswith('multipart/'): continue
                                msg = dict(
                                    headers=dict(msg_hdrs, **part['headers']),
                                    message_id=part['message_id'],
                                    in_reply_to=part['in_reply_to'],
                                    references=part['references'],
                                    filename=part['filename'],
                                    content_type=part['content_type'],
                                    payload=part['payload'])
                                c.app.handle_message(userpart, msg)
                        else:
                            c.app.handle_message(userpart, msg)
            except exc.MailError, e:
                log.error('Error routing email to %s: %s', addr, e)
            except:
                log.exception('Error routing mail to %s', addr)

@task
def sendmail(fromaddr, destinations, text, reply_to, subject,
             message_id, in_reply_to=None):
    from allura import model as M
    addrs_plain = []
    addrs_html = []
    addrs_multi = []
    if fromaddr is None:
        fromaddr = u'noreply@in.sf.net'
    elif '@' not in fromaddr:
        log.warning('Looking up user with fromaddr: %s', fromaddr)
        user = M.User.query.get(_id=ObjectId(fromaddr), disabled=False)
        if not user:
            log.warning('Cannot find user with ID: %s', fromaddr)
            fromaddr = u'noreply@in.sf.net'
        else:
            fromaddr = user.email_address_header()
    # Divide addresses based on preferred email formats
    for addr in destinations:
        if mail_util.isvalid(addr):
            addrs_plain.append(addr)
        else:
            try:
                user = M.User.query.get(_id=ObjectId(addr), disabled=False)
                if not user:
                    log.warning('Cannot find user with ID: %s', addr)
                    continue
            except:
                log.exception('Error looking up user with ID: %r' % addr)
                continue
            addr = user.email_address_header()
            if not addr and user.email_addresses:
                addr = user.email_addresses[0]
                log.warning('User %s has not set primary email address, using %s',
                            user._id, addr)
            if not addr:
                log.error("User %s (%s) has not set any email address, can't deliver",
                          user._id, user.username)
                continue
            if user.get_pref('email_format') == 'plain':
                addrs_plain.append(addr)
            elif user.get_pref('email_format') == 'html':
                addrs_html.append(addr)
            else:
                addrs_multi.append(addr)
    htmlparser = HTMLParser.HTMLParser()
    plain_msg = mail_util.encode_email_part(htmlparser.unescape(text), 'plain')
    html_text = g.forge_markdown(email=True).convert(text)
    html_msg = mail_util.encode_email_part(html_text, 'html')
    multi_msg = mail_util.make_multipart_message(plain_msg, html_msg)
    smtp_client.sendmail(
        addrs_multi, fromaddr, reply_to, subject, message_id,
        in_reply_to, multi_msg)
    smtp_client.sendmail(
        addrs_plain, fromaddr, reply_to, subject, message_id,
        in_reply_to, plain_msg)
    smtp_client.sendmail(
        addrs_html, fromaddr, reply_to, subject, message_id,
        in_reply_to, html_msg)

@task
def sendsimplemail(
    fromaddr,
    toaddr,
    text,
    reply_to,
    subject,
    message_id,
    in_reply_to=None):
    from allura import model as M
    if fromaddr is None:
        fromaddr = u'noreply@in.sf.net'
    elif '@' not in fromaddr:
        log.warning('Looking up user with fromaddr: %s', fromaddr)
        user = M.User.query.get(_id=ObjectId(fromaddr), disabled=False)
        if not user:
            log.warning('Cannot find user with ID: %s', fromaddr)
            fromaddr = u'noreply@in.sf.net'
        else:
            fromaddr = user.email_address_header()
    htmlparser = HTMLParser.HTMLParser()
    plain_msg = mail_util.encode_email_part(htmlparser.unescape(text), 'plain')
    html_text = g.forge_markdown(email=True).convert(text)
    html_msg = mail_util.encode_email_part(html_text, 'html')
    multi_msg = mail_util.make_multipart_message(plain_msg, html_msg)
    smtp_client.sendmail(
        [toaddr], fromaddr, reply_to, subject, message_id,
        in_reply_to, multi_msg)
<|endoftext|>


<commit_before>import py
from py.__.code._assertionold import View
def exvalue():
    return py.std.sys.exc_info()[1]

def setup_module(mod):
    py.code.patch_builtins(assertion=True, compile=False)

def teardown_module(mod):
    py.code.unpatch_builtins(assertion=True, compile=False)

def f():
    return 2

def test_assert():
    try:
        assert f() == 3
    except AssertionError:
        e = exvalue()
        s = str(e)
        assert s.startswith('assert 2 == 3\n')

def test_assert_with_explicit_message():
    try:
        assert f() == 3, "hello"
    except AssertionError:
        e = exvalue()
        assert e.msg == 'hello'

def test_assert_within_finally():
    class A:
        def f():
            pass
    excinfo = py.test.raises(TypeError, """
        try:
            A().f()
        finally:
            i = 42
    """)
    s = excinfo.exconly()
    assert s.find("takes no argument") != -1

    #def g():
    #    A.f()
    #excinfo = getexcinfo(TypeError, g)
    #msg = getmsg(excinfo)
    #assert msg.find("must be called with A") != -1

def test_assert_multiline_1():
    try:
        assert (f() ==
                3)
    except AssertionError:
        e = exvalue()
        s = str(e)
        assert s.startswith('assert 2 == 3\n')

def test_assert_multiline_2():
    try:
        assert (f() == (4,
                   3)[-1])
    except AssertionError:
        e = exvalue()
        s = str(e)
        assert s.startswith('assert 2 ==')

def test_assert_non_string_message():
    class A:
        def __str__(self):
            return "hello"
    try:
        assert 0 == 1, A()
    except AssertionError:
        e = exvalue()
        assert e.msg == "hello"

# These tests should both fail, but should fail nicely...
class WeirdRepr:
    def __repr__(self):
        return '<WeirdRepr\nsecond line>'

def bug_test_assert_repr():
    v = WeirdRepr()
    try:
        assert v == 1
    except AssertionError:
        e = exvalue()
        assert e.msg.find('WeirdRepr') != -1
        assert e.msg.find('second line') != -1
        assert 0

def test_assert_non_string():
    try:
        assert 0, ['list']
    except AssertionError:
        e = exvalue()
        assert e.msg.find("list") != -1

def test_assert_implicit_multiline():
    try:
        x = [1,2,3]
        assert x != [1,
           2, 3]
    except AssertionError:
        e = exvalue()
        assert e.msg.find('assert [1, 2, 3] !=') != -1

def test_assert_with_brokenrepr_arg():
    class BrokenRepr:
        def __repr__(self): 0 / 0
    e = AssertionError(BrokenRepr())
    if e.msg.find("broken __repr__") == -1:
        py.test.fail("broken __repr__ not handle correctly")

class TestView:
    def test_class_dispatch(self):
        ### Use a custom class hierarchy with existing instances

        class Picklable(View):
            pass

        class Simple(Picklable):
            __view__ = object
            def pickle(self):
                return repr(self.__obj__)

        class Seq(Picklable):
            __view__ = list, tuple, dict
            def pickle(self):
                return ';'.join(
                    [Picklable(item).pickle() for item in self.__obj__])

        class Dict(Seq):
            __view__ = dict
            def pickle(self):
                return Seq.pickle(self) + '!' + Seq(self.values()).pickle()

        assert Picklable(123).pickle() == '123'
        assert Picklable([1,[2,3],4]).pickle() == '1;2;3;4'
        assert Picklable({1:2}).pickle() == '1!2'

    def test_viewtype_class_hierarchy(self):
        # Use a custom class hierarchy based on attributes of existing instances
        class Operation:
            "Existing class that I don't want to change."
            def __init__(self, opname, *args):
                self.opname = opname
                self.args = args

        existing = [Operation('+', 4, 5),
                    Operation('getitem', '', 'join'),
                    Operation('setattr', 'x', 'y', 3),
                    Operation('-', 12, 1)]

        class PyOp(View):
            def __viewkey__(self):
                return self.opname
            def generate(self):
                return '%s(%s)' % (self.opname, ', '.join(map(repr, self.args)))

        class PyBinaryOp(PyOp):
            __view__ = ('+', '-', '*', '/')
            def generate(self):
                return '%s %s %s' % (self.args[0], self.opname, self.args[1])

        codelines = [PyOp(op).generate() for op in existing]
        assert codelines == ["4 + 5", "getitem('', 'join')",
            "setattr('x', 'y', 3)", "12 - 1"]

<commit_msg>only test View on 2.x<commit_after>import py

def exvalue():
    return py.std.sys.exc_info()[1]

def setup_module(mod):
    py.code.patch_builtins(assertion=True, compile=False)

def teardown_module(mod):
    py.code.unpatch_builtins(assertion=True, compile=False)

def f():
    return 2

def test_assert():
    try:
        assert f() == 3
    except AssertionError:
        e = exvalue()
        s = str(e)
        assert s.startswith('assert 2 == 3\n')

def test_assert_with_explicit_message():
    try:
        assert f() == 3, "hello"
    except AssertionError:
        e = exvalue()
        assert e.msg == 'hello'

def test_assert_within_finally():
    class A:
        def f():
            pass
    excinfo = py.test.raises(TypeError, """
        try:
            A().f()
        finally:
            i = 42
    """)
    s = excinfo.exconly()
    assert s.find("takes no argument") != -1

    #def g():
    #    A.f()
    #excinfo = getexcinfo(TypeError, g)
    #msg = getmsg(excinfo)
    #assert msg.find("must be called with A") != -1

def test_assert_multiline_1():
    try:
        assert (f() ==
                3)
    except AssertionError:
        e = exvalue()
        s = str(e)
        assert s.startswith('assert 2 == 3\n')

def test_assert_multiline_2():
    try:
        assert (f() == (4,
                   3)[-1])
    except AssertionError:
        e = exvalue()
        s = str(e)
        assert s.startswith('assert 2 ==')

def test_assert_non_string_message():
    class A:
        def __str__(self):
            return "hello"
    try:
        assert 0 == 1, A()
    except AssertionError:
        e = exvalue()
        assert e.msg == "hello"

# These tests should both fail, but should fail nicely...
class WeirdRepr:
    def __repr__(self):
        return '<WeirdRepr\nsecond line>'

def bug_test_assert_repr():
    v = WeirdRepr()
    try:
        assert v == 1
    except AssertionError:
        e = exvalue()
        assert e.msg.find('WeirdRepr') != -1
        assert e.msg.find('second line') != -1
        assert 0

def test_assert_non_string():
    try:
        assert 0, ['list']
    except AssertionError:
        e = exvalue()
        assert e.msg.find("list") != -1

def test_assert_implicit_multiline():
    try:
        x = [1,2,3]
        assert x != [1,
           2, 3]
    except AssertionError:
        e = exvalue()
        assert e.msg.find('assert [1, 2, 3] !=') != -1

def test_assert_with_brokenrepr_arg():
    class BrokenRepr:
        def __repr__(self): 0 / 0
    e = AssertionError(BrokenRepr())
    if e.msg.find("broken __repr__") == -1:
        py.test.fail("broken __repr__ not handle correctly")

class TestView:

    def setup_class(cls):
        try:
            from py.__.code._assertionold import View
        except ImportError:
            py.test.skip("requires the compile package")
        cls.View = View

    def test_class_dispatch(self):
        ### Use a custom class hierarchy with existing instances

        class Picklable(self.View):
            pass

        class Simple(Picklable):
            __view__ = object
            def pickle(self):
                return repr(self.__obj__)

        class Seq(Picklable):
            __view__ = list, tuple, dict
            def pickle(self):
                return ';'.join(
                    [Picklable(item).pickle() for item in self.__obj__])

        class Dict(Seq):
            __view__ = dict
            def pickle(self):
                return Seq.pickle(self) + '!' + Seq(self.values()).pickle()

        assert Picklable(123).pickle() == '123'
        assert Picklable([1,[2,3],4]).pickle() == '1;2;3;4'
        assert Picklable({1:2}).pickle() == '1!2'

    def test_viewtype_class_hierarchy(self):
        # Use a custom class hierarchy based on attributes of existing instances
        class Operation:
            "Existing class that I don't want to change."
            def __init__(self, opname, *args):
                self.opname = opname
                self.args = args

        existing = [Operation('+', 4, 5),
                    Operation('getitem', '', 'join'),
                    Operation('setattr', 'x', 'y', 3),
                    Operation('-', 12, 1)]

        class PyOp(self.View):
            def __viewkey__(self):
                return self.opname
            def generate(self):
                return '%s(%s)' % (self.opname, ', '.join(map(repr, self.args)))

        class PyBinaryOp(PyOp):
            __view__ = ('+', '-', '*', '/')
            def generate(self):
                return '%s %s %s' % (self.args[0], self.opname, self.args[1])

        codelines = [PyOp(op).generate() for op in existing]
        assert codelines == ["4 + 5", "getitem('', 'join')",
            "setattr('x', 'y', 3)", "12 - 1"]

<|endoftext|>